Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

Rev

Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
14 pmbaty 1
//===- TargetSelectionDAG.td - Common code for DAG isels ---*- tablegen -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines the target-independent interfaces used by SelectionDAG
10
// instruction selection generators.
11
//
12
//===----------------------------------------------------------------------===//
13
 
14
//===----------------------------------------------------------------------===//
15
// Selection DAG Type Constraint definitions.
16
//
17
// Note that the semantics of these constraints are hard coded into tblgen.  To
18
// modify or add constraints, you have to hack tblgen.
19
//
20
 
21
class SDTypeConstraint<int opnum> {
22
  int OperandNum = opnum;
23
}
24
 
25
// SDTCisVT - The specified operand has exactly this VT.
26
class SDTCisVT<int OpNum, ValueType vt> : SDTypeConstraint<OpNum> {
27
  ValueType VT = vt;
28
}
29
 
30
class SDTCisPtrTy<int OpNum> : SDTypeConstraint<OpNum>;
31
 
32
// SDTCisInt - The specified operand has integer type.
33
class SDTCisInt<int OpNum> : SDTypeConstraint<OpNum>;
34
 
35
// SDTCisFP - The specified operand has floating-point type.
36
class SDTCisFP<int OpNum> : SDTypeConstraint<OpNum>;
37
 
38
// SDTCisVec - The specified operand has a vector type.
39
class SDTCisVec<int OpNum> : SDTypeConstraint<OpNum>;
40
 
41
// SDTCisSameAs - The two specified operands have identical types.
42
class SDTCisSameAs<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> {
43
  int OtherOperandNum = OtherOp;
44
}
45
 
46
// SDTCisVTSmallerThanOp - The specified operand is a VT SDNode, and its type is
47
// smaller than the 'Other' operand.
48
class SDTCisVTSmallerThanOp<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> {
49
  int OtherOperandNum = OtherOp;
50
}
51
 
52
class SDTCisOpSmallerThanOp<int SmallOp, int BigOp> : SDTypeConstraint<SmallOp>{
53
  int BigOperandNum = BigOp;
54
}
55
 
56
/// SDTCisEltOfVec - This indicates that ThisOp is a scalar type of the same
57
/// type as the element type of OtherOp, which is a vector type.
58
class SDTCisEltOfVec<int ThisOp, int OtherOp>
59
  : SDTypeConstraint<ThisOp> {
60
  int OtherOpNum = OtherOp;
61
}
62
 
63
/// SDTCisSubVecOfVec - This indicates that ThisOp is a vector type
64
/// with length less that of OtherOp, which is a vector type.
65
class SDTCisSubVecOfVec<int ThisOp, int OtherOp>
66
  : SDTypeConstraint<ThisOp> {
67
  int OtherOpNum = OtherOp;
68
}
69
 
70
// SDTCVecEltisVT - The specified operand is vector type with element type
71
// of VT.
72
class SDTCVecEltisVT<int OpNum, ValueType vt> : SDTypeConstraint<OpNum> {
73
  ValueType VT = vt;
74
}
75
 
76
// SDTCisSameNumEltsAs - The two specified operands have identical number
77
// of elements.
78
class SDTCisSameNumEltsAs<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> {
79
  int OtherOperandNum = OtherOp;
80
}
81
 
82
// SDTCisSameSizeAs - The two specified operands have identical size.
83
class SDTCisSameSizeAs<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> {
84
  int OtherOperandNum = OtherOp;
85
}
86
 
87
//===----------------------------------------------------------------------===//
88
// Selection DAG Type Profile definitions.
89
//
90
// These use the constraints defined above to describe the type requirements of
91
// the various nodes.  These are not hard coded into tblgen, allowing targets to
92
// add their own if needed.
93
//
94
 
95
// SDTypeProfile - This profile describes the type requirements of a Selection
96
// DAG node.
97
class SDTypeProfile<int numresults, int numoperands,
98
                    list<SDTypeConstraint> constraints> {
99
  int NumResults = numresults;
100
  int NumOperands = numoperands;
101
  list<SDTypeConstraint> Constraints = constraints;
102
}
103
 
104
// Builtin profiles.
105
def SDTIntLeaf: SDTypeProfile<1, 0, [SDTCisInt<0>]>;         // for 'imm'.
106
def SDTFPLeaf : SDTypeProfile<1, 0, [SDTCisFP<0>]>;          // for 'fpimm'.
107
def SDTPtrLeaf: SDTypeProfile<1, 0, [SDTCisPtrTy<0>]>;       // for '&g'.
108
def SDTOther  : SDTypeProfile<1, 0, [SDTCisVT<0, OtherVT>]>; // for 'vt'.
109
def SDTUNDEF  : SDTypeProfile<1, 0, []>;                     // for 'undef'.
110
def SDTUnaryOp  : SDTypeProfile<1, 1, []>;                   // for bitconvert.
111
 
112
def SDTPtrAddOp : SDTypeProfile<1, 2, [     // ptradd
113
  SDTCisSameAs<0, 1>, SDTCisInt<2>, SDTCisPtrTy<1>
114
]>;
115
def SDTIntBinOp : SDTypeProfile<1, 2, [     // add, and, or, xor, udiv, etc.
116
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>
117
]>;
118
def SDTIntShiftOp : SDTypeProfile<1, 2, [   // shl, sra, srl
119
  SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisInt<2>
120
]>;
121
def SDTIntShiftDOp: SDTypeProfile<1, 3, [   // fshl, fshr
122
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>, SDTCisInt<3>
123
]>;
124
def SDTIntSatNoShOp : SDTypeProfile<1, 2, [   // ssat with no shift
125
  SDTCisSameAs<0, 1>, SDTCisInt<2>
126
]>;
127
def SDTIntBinHiLoOp : SDTypeProfile<2, 2, [ // mulhi, mullo, sdivrem, udivrem
128
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,SDTCisInt<0>
129
]>;
130
def SDTIntScaledBinOp : SDTypeProfile<1, 3, [  // smulfix, sdivfix, etc
131
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>, SDTCisInt<3>
132
]>;
133
 
134
def SDTFPBinOp : SDTypeProfile<1, 2, [      // fadd, fmul, etc.
135
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisFP<0>
136
]>;
137
def SDTFPSignOp : SDTypeProfile<1, 2, [     // fcopysign.
138
  SDTCisSameAs<0, 1>, SDTCisFP<0>, SDTCisFP<2>
139
]>;
140
def SDTFPTernaryOp : SDTypeProfile<1, 3, [  // fmadd, fnmsub, etc.
141
  SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisFP<0>
142
]>;
143
def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // bitreverse
144
  SDTCisSameAs<0, 1>, SDTCisInt<0>
145
]>;
146
def SDTIntBitCountUnaryOp : SDTypeProfile<1, 1, [   // ctlz, cttz
147
  SDTCisInt<0>, SDTCisInt<1>
148
]>;
149
def SDTIntExtendOp : SDTypeProfile<1, 1, [  // sext, zext, anyext
150
  SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<1, 0>, SDTCisSameNumEltsAs<0, 1>
151
]>;
152
def SDTIntTruncOp  : SDTypeProfile<1, 1, [  // trunc
153
  SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
154
]>;
155
def SDTFPUnaryOp  : SDTypeProfile<1, 1, [   // fneg, fsqrt, etc
156
  SDTCisSameAs<0, 1>, SDTCisFP<0>
157
]>;
158
def SDTFPRoundOp  : SDTypeProfile<1, 1, [   // fpround
159
  SDTCisFP<0>, SDTCisFP<1>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
160
]>;
161
def SDTFPExtendOp  : SDTypeProfile<1, 1, [  // fpextend
162
  SDTCisFP<0>, SDTCisFP<1>, SDTCisOpSmallerThanOp<1, 0>, SDTCisSameNumEltsAs<0, 1>
163
]>;
164
def SDIsFPClassOp : SDTypeProfile<1, 2, [   // is_fpclass
165
  SDTCisInt<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisSameNumEltsAs<0, 1>
166
]>;
167
def SDTIntToFPOp : SDTypeProfile<1, 1, [    // [su]int_to_fp
168
  SDTCisFP<0>, SDTCisInt<1>, SDTCisSameNumEltsAs<0, 1>
169
]>;
170
def SDTFPToIntOp : SDTypeProfile<1, 1, [    // fp_to_[su]int
171
  SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>
172
]>;
173
def SDTFPToIntSatOp : SDTypeProfile<1, 2, [    // fp_to_[su]int_sat
174
  SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>, SDTCisVT<2, OtherVT>
175
]>;
176
def SDTExtInreg : SDTypeProfile<1, 2, [     // sext_inreg
177
  SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisVT<2, OtherVT>,
178
  SDTCisVTSmallerThanOp<2, 1>
179
]>;
180
def SDTExtInvec : SDTypeProfile<1, 1, [     // sext_invec
181
  SDTCisInt<0>, SDTCisVec<0>, SDTCisInt<1>, SDTCisVec<1>,
182
  SDTCisOpSmallerThanOp<1, 0>
183
]>;
184
def SDTFreeze : SDTypeProfile<1, 1, [
185
  SDTCisSameAs<0, 1>
186
]>;
187
 
188
def SDTSetCC : SDTypeProfile<1, 3, [        // setcc
189
  SDTCisInt<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, OtherVT>
190
]>;
191
 
192
def SDTSelect : SDTypeProfile<1, 3, [       // select
193
  SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>
194
]>;
195
 
196
def SDTVSelect : SDTypeProfile<1, 3, [       // vselect
197
  SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>, SDTCisSameNumEltsAs<0, 1>
198
]>;
199
 
200
def SDTSelectCC : SDTypeProfile<1, 5, [     // select_cc
201
  SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>, SDTCisSameAs<0, 3>,
202
  SDTCisVT<5, OtherVT>
203
]>;
204
 
205
def SDTBr : SDTypeProfile<0, 1, [           // br
206
  SDTCisVT<0, OtherVT>
207
]>;
208
 
209
def SDTBrCC : SDTypeProfile<0, 4, [       // brcc
210
  SDTCisVT<0, OtherVT>, SDTCisSameAs<1, 2>, SDTCisVT<3, OtherVT>
211
]>;
212
 
213
def SDTBrcond : SDTypeProfile<0, 2, [       // brcond
214
  SDTCisInt<0>, SDTCisVT<1, OtherVT>
215
]>;
216
 
217
def SDTBrind : SDTypeProfile<0, 1, [        // brind
218
  SDTCisPtrTy<0>
219
]>;
220
 
221
def SDTCatchret : SDTypeProfile<0, 2, [     // catchret
222
  SDTCisVT<0, OtherVT>, SDTCisVT<1, OtherVT>
223
]>;
224
 
225
def SDTNone : SDTypeProfile<0, 0, []>;      // ret, trap
226
 
227
def SDTUBSANTrap : SDTypeProfile<0, 1, []>;      // ubsantrap
228
 
229
def SDTLoad : SDTypeProfile<1, 1, [         // load
230
  SDTCisPtrTy<1>
231
]>;
232
 
233
def SDTStore : SDTypeProfile<0, 2, [        // store
234
  SDTCisPtrTy<1>
235
]>;
236
 
237
def SDTIStore : SDTypeProfile<1, 3, [       // indexed store
238
  SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3>
239
]>;
240
 
241
def SDTMaskedStore: SDTypeProfile<0, 4, [       // masked store
242
  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameNumEltsAs<0, 3>
243
]>;
244
 
245
def SDTMaskedLoad: SDTypeProfile<1, 4, [       // masked load
246
  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameAs<0, 4>,
247
  SDTCisSameNumEltsAs<0, 3>
248
]>;
249
 
250
def SDTMaskedGather : SDTypeProfile<1, 4, [
251
  SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVec<2>, SDTCisPtrTy<3>, SDTCisVec<4>,
252
  SDTCisSameNumEltsAs<0, 2>, SDTCisSameNumEltsAs<0, 4>
253
]>;
254
 
255
def SDTMaskedScatter : SDTypeProfile<0, 4, [
256
  SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>,
257
  SDTCisSameNumEltsAs<0, 1>, SDTCisSameNumEltsAs<0, 3>
258
]>;
259
 
260
def SDTVecShuffle : SDTypeProfile<1, 2, [
261
  SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
262
]>;
263
def SDTVecSlice : SDTypeProfile<1, 3, [     // vector splice
264
  SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisInt<3>
265
]>;
266
def SDTVecExtract : SDTypeProfile<1, 2, [   // vector extract
267
  SDTCisEltOfVec<0, 1>, SDTCisPtrTy<2>
268
]>;
269
def SDTVecInsert : SDTypeProfile<1, 3, [    // vector insert
270
  SDTCisEltOfVec<2, 1>, SDTCisSameAs<0, 1>, SDTCisPtrTy<3>
271
]>;
272
def SDTVecReduce : SDTypeProfile<1, 1, [    // vector reduction
273
  SDTCisInt<0>, SDTCisVec<1>
274
]>;
275
def SDTFPVecReduce : SDTypeProfile<1, 1, [  // FP vector reduction
276
  SDTCisFP<0>, SDTCisVec<1>
277
]>;
278
 
279
def SDTVecReverse : SDTypeProfile<1, 1, [  // vector reverse
280
  SDTCisVec<0>, SDTCisSameAs<0,1>
281
]>;
282
 
283
def SDTSubVecExtract : SDTypeProfile<1, 2, [// subvector extract
284
  SDTCisSubVecOfVec<0,1>, SDTCisInt<2>
285
]>;
286
def SDTSubVecInsert : SDTypeProfile<1, 3, [ // subvector insert
287
  SDTCisSubVecOfVec<2, 1>, SDTCisSameAs<0,1>, SDTCisInt<3>
288
]>;
289
 
290
def SDTPrefetch : SDTypeProfile<0, 4, [     // prefetch
291
  SDTCisPtrTy<0>, SDTCisSameAs<1, 2>, SDTCisSameAs<1, 3>, SDTCisInt<1>
292
]>;
293
 
294
def SDTAtomicFence : SDTypeProfile<0, 2, [
295
  SDTCisSameAs<0,1>, SDTCisPtrTy<0>
296
]>;
297
def SDTAtomic3 : SDTypeProfile<1, 3, [
298
  SDTCisSameAs<0,2>,  SDTCisSameAs<0,3>, SDTCisInt<0>, SDTCisPtrTy<1>
299
]>;
300
def SDTAtomic2 : SDTypeProfile<1, 2, [
301
  SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1>
302
]>;
303
 
304
def SDTFPAtomic2 : SDTypeProfile<1, 2, [
305
  SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
306
]>;
307
 
308
def SDTAtomicStore : SDTypeProfile<0, 2, [
309
  SDTCisPtrTy<0>, SDTCisInt<1>
310
]>;
311
def SDTAtomicLoad : SDTypeProfile<1, 1, [
312
  SDTCisInt<0>, SDTCisPtrTy<1>
313
]>;
314
 
315
class SDCallSeqStart<list<SDTypeConstraint> constraints> :
316
        SDTypeProfile<0, 2, constraints>;
317
class SDCallSeqEnd<list<SDTypeConstraint> constraints> :
318
        SDTypeProfile<0, 2, constraints>;
319
 
320
//===----------------------------------------------------------------------===//
321
// Selection DAG Node definitions.
322
//
323
class SDNode<string opcode, SDTypeProfile typeprof,
324
             list<SDNodeProperty> props = [], string sdclass = "SDNode">
325
             : SDPatternOperator {
326
  string Opcode  = opcode;
327
  string SDClass = sdclass;
328
  let Properties = props;
329
  SDTypeProfile TypeProfile = typeprof;
330
}
331
 
332
// Special TableGen-recognized dag nodes
333
def set;
334
def implicit;
335
def node;
336
def srcvalue;
337
 
338
def imm        : SDNode<"ISD::Constant"  , SDTIntLeaf , [], "ConstantSDNode">;
339
def timm       : SDNode<"ISD::TargetConstant",SDTIntLeaf, [], "ConstantSDNode">;
340
def fpimm      : SDNode<"ISD::ConstantFP", SDTFPLeaf  , [], "ConstantFPSDNode">;
341
def vt         : SDNode<"ISD::VALUETYPE" , SDTOther   , [], "VTSDNode">;
342
def bb         : SDNode<"ISD::BasicBlock", SDTOther   , [], "BasicBlockSDNode">;
343
def cond       : SDNode<"ISD::CONDCODE"  , SDTOther   , [], "CondCodeSDNode">;
344
def undef      : SDNode<"ISD::UNDEF"     , SDTUNDEF   , []>;
345
def vscale     : SDNode<"ISD::VSCALE"    , SDTIntUnaryOp, []>;
346
def globaladdr : SDNode<"ISD::GlobalAddress",         SDTPtrLeaf, [],
347
                        "GlobalAddressSDNode">;
348
def tglobaladdr : SDNode<"ISD::TargetGlobalAddress",  SDTPtrLeaf, [],
349
                         "GlobalAddressSDNode">;
350
def globaltlsaddr : SDNode<"ISD::GlobalTLSAddress",         SDTPtrLeaf, [],
351
                          "GlobalAddressSDNode">;
352
def tglobaltlsaddr : SDNode<"ISD::TargetGlobalTLSAddress",  SDTPtrLeaf, [],
353
                           "GlobalAddressSDNode">;
354
def constpool   : SDNode<"ISD::ConstantPool",         SDTPtrLeaf, [],
355
                         "ConstantPoolSDNode">;
356
def tconstpool  : SDNode<"ISD::TargetConstantPool",   SDTPtrLeaf, [],
357
                         "ConstantPoolSDNode">;
358
def jumptable   : SDNode<"ISD::JumpTable",            SDTPtrLeaf, [],
359
                         "JumpTableSDNode">;
360
def tjumptable  : SDNode<"ISD::TargetJumpTable",      SDTPtrLeaf, [],
361
                         "JumpTableSDNode">;
362
def frameindex  : SDNode<"ISD::FrameIndex",           SDTPtrLeaf, [],
363
                         "FrameIndexSDNode">;
364
def tframeindex : SDNode<"ISD::TargetFrameIndex",     SDTPtrLeaf, [],
365
                         "FrameIndexSDNode">;
366
def externalsym : SDNode<"ISD::ExternalSymbol",       SDTPtrLeaf, [],
367
                         "ExternalSymbolSDNode">;
368
def texternalsym: SDNode<"ISD::TargetExternalSymbol", SDTPtrLeaf, [],
369
                         "ExternalSymbolSDNode">;
370
def mcsym: SDNode<"ISD::MCSymbol", SDTPtrLeaf, [], "MCSymbolSDNode">;
371
def blockaddress : SDNode<"ISD::BlockAddress",        SDTPtrLeaf, [],
372
                         "BlockAddressSDNode">;
373
def tblockaddress: SDNode<"ISD::TargetBlockAddress",  SDTPtrLeaf, [],
374
                         "BlockAddressSDNode">;
375
 
376
def add        : SDNode<"ISD::ADD"       , SDTIntBinOp   ,
377
                        [SDNPCommutative, SDNPAssociative]>;
378
def ptradd     : SDNode<"ISD::ADD"       , SDTPtrAddOp, []>;
379
def sub        : SDNode<"ISD::SUB"       , SDTIntBinOp>;
380
def mul        : SDNode<"ISD::MUL"       , SDTIntBinOp,
381
                        [SDNPCommutative, SDNPAssociative]>;
382
def mulhs      : SDNode<"ISD::MULHS"     , SDTIntBinOp, [SDNPCommutative]>;
383
def mulhu      : SDNode<"ISD::MULHU"     , SDTIntBinOp, [SDNPCommutative]>;
384
def avgfloors  : SDNode<"ISD::AVGFLOORS" , SDTIntBinOp, [SDNPCommutative]>;
385
def avgflooru  : SDNode<"ISD::AVGFLOORU" , SDTIntBinOp, [SDNPCommutative]>;
386
def avgceils   : SDNode<"ISD::AVGCEILS"  , SDTIntBinOp, [SDNPCommutative]>;
387
def avgceilu   : SDNode<"ISD::AVGCEILU"  , SDTIntBinOp, [SDNPCommutative]>;
388
def abds       : SDNode<"ISD::ABDS"      , SDTIntBinOp, [SDNPCommutative]>;
389
def abdu       : SDNode<"ISD::ABDU"      , SDTIntBinOp, [SDNPCommutative]>;
390
def smullohi   : SDNode<"ISD::SMUL_LOHI" , SDTIntBinHiLoOp, [SDNPCommutative]>;
391
def umullohi   : SDNode<"ISD::UMUL_LOHI" , SDTIntBinHiLoOp, [SDNPCommutative]>;
392
def sdiv       : SDNode<"ISD::SDIV"      , SDTIntBinOp>;
393
def udiv       : SDNode<"ISD::UDIV"      , SDTIntBinOp>;
394
def srem       : SDNode<"ISD::SREM"      , SDTIntBinOp>;
395
def urem       : SDNode<"ISD::UREM"      , SDTIntBinOp>;
396
def sdivrem    : SDNode<"ISD::SDIVREM"   , SDTIntBinHiLoOp>;
397
def udivrem    : SDNode<"ISD::UDIVREM"   , SDTIntBinHiLoOp>;
398
def srl        : SDNode<"ISD::SRL"       , SDTIntShiftOp>;
399
def sra        : SDNode<"ISD::SRA"       , SDTIntShiftOp>;
400
def shl        : SDNode<"ISD::SHL"       , SDTIntShiftOp>;
401
def rotl       : SDNode<"ISD::ROTL"      , SDTIntShiftOp>;
402
def rotr       : SDNode<"ISD::ROTR"      , SDTIntShiftOp>;
403
def fshl       : SDNode<"ISD::FSHL"      , SDTIntShiftDOp>;
404
def fshr       : SDNode<"ISD::FSHR"      , SDTIntShiftDOp>;
405
def and        : SDNode<"ISD::AND"       , SDTIntBinOp,
406
                        [SDNPCommutative, SDNPAssociative]>;
407
def or         : SDNode<"ISD::OR"        , SDTIntBinOp,
408
                        [SDNPCommutative, SDNPAssociative]>;
409
def xor        : SDNode<"ISD::XOR"       , SDTIntBinOp,
410
                        [SDNPCommutative, SDNPAssociative]>;
411
def addc       : SDNode<"ISD::ADDC"      , SDTIntBinOp,
412
                        [SDNPCommutative, SDNPOutGlue]>;
413
def adde       : SDNode<"ISD::ADDE"      , SDTIntBinOp,
414
                        [SDNPCommutative, SDNPOutGlue, SDNPInGlue]>;
415
def subc       : SDNode<"ISD::SUBC"      , SDTIntBinOp,
416
                        [SDNPOutGlue]>;
417
def sube       : SDNode<"ISD::SUBE"      , SDTIntBinOp,
418
                        [SDNPOutGlue, SDNPInGlue]>;
419
def smin       : SDNode<"ISD::SMIN"      , SDTIntBinOp,
420
                                  [SDNPCommutative, SDNPAssociative]>;
421
def smax       : SDNode<"ISD::SMAX"      , SDTIntBinOp,
422
                                  [SDNPCommutative, SDNPAssociative]>;
423
def umin       : SDNode<"ISD::UMIN"      , SDTIntBinOp,
424
                                  [SDNPCommutative, SDNPAssociative]>;
425
def umax       : SDNode<"ISD::UMAX"      , SDTIntBinOp,
426
                                  [SDNPCommutative, SDNPAssociative]>;
427
 
428
def saddsat    : SDNode<"ISD::SADDSAT"   , SDTIntBinOp, [SDNPCommutative]>;
429
def uaddsat    : SDNode<"ISD::UADDSAT"   , SDTIntBinOp, [SDNPCommutative]>;
430
def ssubsat    : SDNode<"ISD::SSUBSAT"   , SDTIntBinOp>;
431
def usubsat    : SDNode<"ISD::USUBSAT"   , SDTIntBinOp>;
432
def sshlsat    : SDNode<"ISD::SSHLSAT"   , SDTIntBinOp>;
433
def ushlsat    : SDNode<"ISD::USHLSAT"   , SDTIntBinOp>;
434
 
435
def smulfix    : SDNode<"ISD::SMULFIX"   , SDTIntScaledBinOp, [SDNPCommutative]>;
436
def smulfixsat : SDNode<"ISD::SMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>;
437
def umulfix    : SDNode<"ISD::UMULFIX"   , SDTIntScaledBinOp, [SDNPCommutative]>;
438
def umulfixsat : SDNode<"ISD::UMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>;
439
def sdivfix    : SDNode<"ISD::SDIVFIX"   , SDTIntScaledBinOp>;
440
def sdivfixsat : SDNode<"ISD::SDIVFIXSAT", SDTIntScaledBinOp>;
441
def udivfix    : SDNode<"ISD::UDIVFIX"   , SDTIntScaledBinOp>;
442
def udivfixsat : SDNode<"ISD::UDIVFIXSAT", SDTIntScaledBinOp>;
443
 
444
def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>;
445
def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>;
446
def zext_invec : SDNode<"ISD::ZERO_EXTEND_VECTOR_INREG", SDTExtInvec>;
447
 
448
def abs        : SDNode<"ISD::ABS"        , SDTIntUnaryOp>;
449
def bitreverse : SDNode<"ISD::BITREVERSE" , SDTIntUnaryOp>;
450
def bswap      : SDNode<"ISD::BSWAP"      , SDTIntUnaryOp>;
451
def ctlz       : SDNode<"ISD::CTLZ"       , SDTIntBitCountUnaryOp>;
452
def cttz       : SDNode<"ISD::CTTZ"       , SDTIntBitCountUnaryOp>;
453
def ctpop      : SDNode<"ISD::CTPOP"      , SDTIntBitCountUnaryOp>;
454
def ctlz_zero_undef : SDNode<"ISD::CTLZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>;
455
def cttz_zero_undef : SDNode<"ISD::CTTZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>;
456
def sext       : SDNode<"ISD::SIGN_EXTEND", SDTIntExtendOp>;
457
def zext       : SDNode<"ISD::ZERO_EXTEND", SDTIntExtendOp>;
458
def anyext     : SDNode<"ISD::ANY_EXTEND" , SDTIntExtendOp>;
459
def trunc      : SDNode<"ISD::TRUNCATE"   , SDTIntTruncOp>;
460
def bitconvert : SDNode<"ISD::BITCAST"    , SDTUnaryOp>;
461
def addrspacecast : SDNode<"ISD::ADDRSPACECAST", SDTUnaryOp>;
462
def freeze     : SDNode<"ISD::FREEZE"     , SDTFreeze>;
463
def extractelt : SDNode<"ISD::EXTRACT_VECTOR_ELT", SDTVecExtract>;
464
def insertelt  : SDNode<"ISD::INSERT_VECTOR_ELT", SDTVecInsert>;
465
 
466
def vecreduce_add  : SDNode<"ISD::VECREDUCE_ADD", SDTVecReduce>;
467
def vecreduce_smax  : SDNode<"ISD::VECREDUCE_SMAX", SDTVecReduce>;
468
def vecreduce_umax  : SDNode<"ISD::VECREDUCE_UMAX", SDTVecReduce>;
469
def vecreduce_smin  : SDNode<"ISD::VECREDUCE_SMIN", SDTVecReduce>;
470
def vecreduce_umin  : SDNode<"ISD::VECREDUCE_UMIN", SDTVecReduce>;
471
def vecreduce_fadd  : SDNode<"ISD::VECREDUCE_FADD", SDTFPVecReduce>;
472
 
473
def fadd       : SDNode<"ISD::FADD"       , SDTFPBinOp, [SDNPCommutative]>;
474
def fsub       : SDNode<"ISD::FSUB"       , SDTFPBinOp>;
475
def fmul       : SDNode<"ISD::FMUL"       , SDTFPBinOp, [SDNPCommutative]>;
476
def fdiv       : SDNode<"ISD::FDIV"       , SDTFPBinOp>;
477
def frem       : SDNode<"ISD::FREM"       , SDTFPBinOp>;
478
def fma        : SDNode<"ISD::FMA"        , SDTFPTernaryOp, [SDNPCommutative]>;
479
def fmad       : SDNode<"ISD::FMAD"       , SDTFPTernaryOp, [SDNPCommutative]>;
480
def fabs       : SDNode<"ISD::FABS"       , SDTFPUnaryOp>;
481
def fminnum    : SDNode<"ISD::FMINNUM"    , SDTFPBinOp,
482
                                  [SDNPCommutative, SDNPAssociative]>;
483
def fmaxnum    : SDNode<"ISD::FMAXNUM"    , SDTFPBinOp,
484
                                  [SDNPCommutative, SDNPAssociative]>;
485
def fminnum_ieee : SDNode<"ISD::FMINNUM_IEEE", SDTFPBinOp,
486
                          [SDNPCommutative]>;
487
def fmaxnum_ieee  : SDNode<"ISD::FMAXNUM_IEEE", SDTFPBinOp,
488
                           [SDNPCommutative]>;
489
def fminimum   : SDNode<"ISD::FMINIMUM"   , SDTFPBinOp,
490
                        [SDNPCommutative, SDNPAssociative]>;
491
def fmaximum   : SDNode<"ISD::FMAXIMUM"   , SDTFPBinOp,
492
                        [SDNPCommutative, SDNPAssociative]>;
493
def fgetsign   : SDNode<"ISD::FGETSIGN"   , SDTFPToIntOp>;
494
def fcanonicalize : SDNode<"ISD::FCANONICALIZE", SDTFPUnaryOp>;
495
def fneg       : SDNode<"ISD::FNEG"       , SDTFPUnaryOp>;
496
def fsqrt      : SDNode<"ISD::FSQRT"      , SDTFPUnaryOp>;
497
def fsin       : SDNode<"ISD::FSIN"       , SDTFPUnaryOp>;
498
def fcos       : SDNode<"ISD::FCOS"       , SDTFPUnaryOp>;
499
def fexp2      : SDNode<"ISD::FEXP2"      , SDTFPUnaryOp>;
500
def fpow       : SDNode<"ISD::FPOW"       , SDTFPBinOp>;
501
def flog2      : SDNode<"ISD::FLOG2"      , SDTFPUnaryOp>;
502
def frint      : SDNode<"ISD::FRINT"      , SDTFPUnaryOp>;
503
def ftrunc     : SDNode<"ISD::FTRUNC"     , SDTFPUnaryOp>;
504
def fceil      : SDNode<"ISD::FCEIL"      , SDTFPUnaryOp>;
505
def ffloor     : SDNode<"ISD::FFLOOR"     , SDTFPUnaryOp>;
506
def fnearbyint : SDNode<"ISD::FNEARBYINT" , SDTFPUnaryOp>;
507
def fround     : SDNode<"ISD::FROUND"     , SDTFPUnaryOp>;
508
def froundeven : SDNode<"ISD::FROUNDEVEN" , SDTFPUnaryOp>;
509
 
510
def lround     : SDNode<"ISD::LROUND"     , SDTFPToIntOp>;
511
def llround    : SDNode<"ISD::LLROUND"    , SDTFPToIntOp>;
512
def lrint      : SDNode<"ISD::LRINT"      , SDTFPToIntOp>;
513
def llrint     : SDNode<"ISD::LLRINT"     , SDTFPToIntOp>;
514
 
515
def fpround    : SDNode<"ISD::FP_ROUND"   , SDTFPRoundOp>;
516
def fpextend   : SDNode<"ISD::FP_EXTEND"  , SDTFPExtendOp>;
517
def fcopysign  : SDNode<"ISD::FCOPYSIGN"  , SDTFPSignOp>;
518
 
519
def is_fpclass : SDNode<"ISD::IS_FPCLASS" , SDIsFPClassOp>;
520
 
521
def sint_to_fp : SDNode<"ISD::SINT_TO_FP" , SDTIntToFPOp>;
522
def uint_to_fp : SDNode<"ISD::UINT_TO_FP" , SDTIntToFPOp>;
523
def fp_to_sint : SDNode<"ISD::FP_TO_SINT" , SDTFPToIntOp>;
524
def fp_to_uint : SDNode<"ISD::FP_TO_UINT" , SDTFPToIntOp>;
525
def fp_to_sint_sat : SDNode<"ISD::FP_TO_SINT_SAT" , SDTFPToIntSatOp>;
526
def fp_to_uint_sat : SDNode<"ISD::FP_TO_UINT_SAT" , SDTFPToIntSatOp>;
527
def f16_to_fp  : SDNode<"ISD::FP16_TO_FP" , SDTIntToFPOp>;
528
def fp_to_f16  : SDNode<"ISD::FP_TO_FP16" , SDTFPToIntOp>;
529
 
530
def strict_fadd       : SDNode<"ISD::STRICT_FADD",
531
                               SDTFPBinOp, [SDNPHasChain, SDNPCommutative]>;
532
def strict_fsub       : SDNode<"ISD::STRICT_FSUB",
533
                               SDTFPBinOp, [SDNPHasChain]>;
534
def strict_fmul       : SDNode<"ISD::STRICT_FMUL",
535
                               SDTFPBinOp, [SDNPHasChain, SDNPCommutative]>;
536
def strict_fdiv       : SDNode<"ISD::STRICT_FDIV",
537
                               SDTFPBinOp, [SDNPHasChain]>;
538
def strict_frem       : SDNode<"ISD::STRICT_FREM",
539
                               SDTFPBinOp, [SDNPHasChain]>;
540
def strict_fma        : SDNode<"ISD::STRICT_FMA",
541
                               SDTFPTernaryOp, [SDNPHasChain, SDNPCommutative]>;
542
def strict_fsqrt      : SDNode<"ISD::STRICT_FSQRT",
543
                               SDTFPUnaryOp, [SDNPHasChain]>;
544
def strict_fsin       : SDNode<"ISD::STRICT_FSIN",
545
                               SDTFPUnaryOp, [SDNPHasChain]>;
546
def strict_fcos       : SDNode<"ISD::STRICT_FCOS",
547
                               SDTFPUnaryOp, [SDNPHasChain]>;
548
def strict_fexp2      : SDNode<"ISD::STRICT_FEXP2",
549
                               SDTFPUnaryOp, [SDNPHasChain]>;
550
def strict_fpow       : SDNode<"ISD::STRICT_FPOW",
551
                               SDTFPBinOp, [SDNPHasChain]>;
552
def strict_flog2      : SDNode<"ISD::STRICT_FLOG2",
553
                               SDTFPUnaryOp, [SDNPHasChain]>;
554
def strict_frint      : SDNode<"ISD::STRICT_FRINT",
555
                               SDTFPUnaryOp, [SDNPHasChain]>;
556
def strict_lrint      : SDNode<"ISD::STRICT_LRINT",
557
                               SDTFPToIntOp, [SDNPHasChain]>;
558
def strict_llrint     : SDNode<"ISD::STRICT_LLRINT",
559
                               SDTFPToIntOp, [SDNPHasChain]>;
560
def strict_fnearbyint : SDNode<"ISD::STRICT_FNEARBYINT",
561
                               SDTFPUnaryOp, [SDNPHasChain]>;
562
def strict_fceil      : SDNode<"ISD::STRICT_FCEIL",
563
                               SDTFPUnaryOp, [SDNPHasChain]>;
564
def strict_ffloor     : SDNode<"ISD::STRICT_FFLOOR",
565
                               SDTFPUnaryOp, [SDNPHasChain]>;
566
def strict_lround     : SDNode<"ISD::STRICT_LROUND",
567
                               SDTFPToIntOp, [SDNPHasChain]>;
568
def strict_llround    : SDNode<"ISD::STRICT_LLROUND",
569
                               SDTFPToIntOp, [SDNPHasChain]>;
570
def strict_fround     : SDNode<"ISD::STRICT_FROUND",
571
                               SDTFPUnaryOp, [SDNPHasChain]>;
572
def strict_froundeven : SDNode<"ISD::STRICT_FROUNDEVEN",
573
                               SDTFPUnaryOp, [SDNPHasChain]>;
574
def strict_ftrunc     : SDNode<"ISD::STRICT_FTRUNC",
575
                               SDTFPUnaryOp, [SDNPHasChain]>;
576
def strict_fminnum    : SDNode<"ISD::STRICT_FMINNUM",
577
                               SDTFPBinOp, [SDNPHasChain,
578
                                            SDNPCommutative, SDNPAssociative]>;
579
def strict_fmaxnum    : SDNode<"ISD::STRICT_FMAXNUM",
580
                               SDTFPBinOp, [SDNPHasChain,
581
                                            SDNPCommutative, SDNPAssociative]>;
582
def strict_fminimum   : SDNode<"ISD::STRICT_FMINIMUM",
583
                               SDTFPBinOp, [SDNPHasChain,
584
                                            SDNPCommutative, SDNPAssociative]>;
585
def strict_fmaximum   : SDNode<"ISD::STRICT_FMAXIMUM",
586
                               SDTFPBinOp, [SDNPHasChain,
587
                                            SDNPCommutative, SDNPAssociative]>;
588
def strict_fpround    : SDNode<"ISD::STRICT_FP_ROUND",
589
                               SDTFPRoundOp, [SDNPHasChain]>;
590
def strict_fpextend   : SDNode<"ISD::STRICT_FP_EXTEND",
591
                               SDTFPExtendOp, [SDNPHasChain]>;
592
def strict_fp_to_sint : SDNode<"ISD::STRICT_FP_TO_SINT",
593
                               SDTFPToIntOp, [SDNPHasChain]>;
594
def strict_fp_to_uint : SDNode<"ISD::STRICT_FP_TO_UINT",
595
                               SDTFPToIntOp, [SDNPHasChain]>;
596
def strict_sint_to_fp : SDNode<"ISD::STRICT_SINT_TO_FP",
597
                               SDTIntToFPOp, [SDNPHasChain]>;
598
def strict_uint_to_fp : SDNode<"ISD::STRICT_UINT_TO_FP",
599
                               SDTIntToFPOp, [SDNPHasChain]>;
600
def strict_fsetcc  : SDNode<"ISD::STRICT_FSETCC",  SDTSetCC, [SDNPHasChain]>;
601
def strict_fsetccs : SDNode<"ISD::STRICT_FSETCCS", SDTSetCC, [SDNPHasChain]>;
602
 
603
def setcc      : SDNode<"ISD::SETCC"      , SDTSetCC>;
604
def select     : SDNode<"ISD::SELECT"     , SDTSelect>;
605
def vselect    : SDNode<"ISD::VSELECT"    , SDTVSelect>;
606
def selectcc   : SDNode<"ISD::SELECT_CC"  , SDTSelectCC>;
607
 
608
def brcc       : SDNode<"ISD::BR_CC"      , SDTBrCC,   [SDNPHasChain]>;
609
def brcond     : SDNode<"ISD::BRCOND"     , SDTBrcond, [SDNPHasChain]>;
610
def brind      : SDNode<"ISD::BRIND"      , SDTBrind,  [SDNPHasChain]>;
611
def br         : SDNode<"ISD::BR"         , SDTBr,     [SDNPHasChain]>;
612
def catchret   : SDNode<"ISD::CATCHRET"   , SDTCatchret,
613
                        [SDNPHasChain, SDNPSideEffect]>;
614
def cleanupret : SDNode<"ISD::CLEANUPRET" , SDTNone,   [SDNPHasChain]>;
615
 
616
def trap       : SDNode<"ISD::TRAP"       , SDTNone,
617
                        [SDNPHasChain, SDNPSideEffect]>;
618
def debugtrap  : SDNode<"ISD::DEBUGTRAP"  , SDTNone,
619
                        [SDNPHasChain, SDNPSideEffect]>;
620
def ubsantrap  : SDNode<"ISD::UBSANTRAP"  , SDTUBSANTrap,
621
                        [SDNPHasChain, SDNPSideEffect]>;
622
 
623
def prefetch   : SDNode<"ISD::PREFETCH"   , SDTPrefetch,
624
                        [SDNPHasChain, SDNPMayLoad, SDNPMayStore,
625
                         SDNPMemOperand]>;
626
 
627
def readcyclecounter : SDNode<"ISD::READCYCLECOUNTER", SDTIntLeaf,
628
                     [SDNPHasChain, SDNPSideEffect]>;
629
 
630
def membarrier : SDNode<"ISD::MEMBARRIER", SDTNone,
631
                        [SDNPHasChain, SDNPSideEffect]>;
632
 
633
def atomic_fence : SDNode<"ISD::ATOMIC_FENCE" , SDTAtomicFence,
634
                          [SDNPHasChain, SDNPSideEffect]>;
635
 
636
def atomic_cmp_swap : SDNode<"ISD::ATOMIC_CMP_SWAP" , SDTAtomic3,
637
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
638
def atomic_load_add : SDNode<"ISD::ATOMIC_LOAD_ADD" , SDTAtomic2,
639
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
640
def atomic_swap     : SDNode<"ISD::ATOMIC_SWAP", SDTAtomic2,
641
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
642
def atomic_load_sub : SDNode<"ISD::ATOMIC_LOAD_SUB" , SDTAtomic2,
643
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
644
def atomic_load_and : SDNode<"ISD::ATOMIC_LOAD_AND" , SDTAtomic2,
645
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
646
def atomic_load_clr : SDNode<"ISD::ATOMIC_LOAD_CLR" , SDTAtomic2,
647
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
648
def atomic_load_or  : SDNode<"ISD::ATOMIC_LOAD_OR" , SDTAtomic2,
649
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
650
def atomic_load_xor : SDNode<"ISD::ATOMIC_LOAD_XOR" , SDTAtomic2,
651
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
652
def atomic_load_nand: SDNode<"ISD::ATOMIC_LOAD_NAND", SDTAtomic2,
653
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
654
def atomic_load_min : SDNode<"ISD::ATOMIC_LOAD_MIN", SDTAtomic2,
655
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
656
def atomic_load_max : SDNode<"ISD::ATOMIC_LOAD_MAX", SDTAtomic2,
657
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
658
def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", SDTAtomic2,
659
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
660
def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,
661
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
662
def atomic_load_fadd : SDNode<"ISD::ATOMIC_LOAD_FADD" , SDTFPAtomic2,
663
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
664
def atomic_load_fsub : SDNode<"ISD::ATOMIC_LOAD_FSUB" , SDTFPAtomic2,
665
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
666
def atomic_load_fmax : SDNode<"ISD::ATOMIC_LOAD_FMAX", SDTFPAtomic2,
667
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
668
def atomic_load_fmin : SDNode<"ISD::ATOMIC_LOAD_FMIN", SDTFPAtomic2,
669
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
670
def atomic_load_uinc_wrap : SDNode<"ISD::ATOMIC_LOAD_UINC_WRAP", SDTAtomic2,
671
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
672
def atomic_load_udec_wrap : SDNode<"ISD::ATOMIC_LOAD_UDEC_WRAP", SDTAtomic2,
673
                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
674
 
675
def atomic_load      : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
676
                    [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
677
def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
678
                    [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
679
 
680
def masked_st    : SDNode<"ISD::MSTORE",  SDTMaskedStore,
681
                       [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
682
def masked_ld    : SDNode<"ISD::MLOAD",  SDTMaskedLoad,
683
                       [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
684
 
685
def masked_gather : SDNode<"ISD::MGATHER", SDTMaskedGather,
686
                           [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
687
 
688
def masked_scatter : SDNode<"ISD::MSCATTER", SDTMaskedScatter,
689
                            [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
690
 
691
// Do not use ld, st directly. Use load, extload, sextload, zextload, store,
692
// and truncst (see below).
693
def ld         : SDNode<"ISD::LOAD"       , SDTLoad,
694
                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
695
def st         : SDNode<"ISD::STORE"      , SDTStore,
696
                        [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
697
def ist        : SDNode<"ISD::STORE"      , SDTIStore,
698
                        [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
699
 
700
def vector_shuffle : SDNode<"ISD::VECTOR_SHUFFLE", SDTVecShuffle, []>;
701
def vector_reverse : SDNode<"ISD::VECTOR_REVERSE", SDTVecReverse>;
702
def vector_splice : SDNode<"ISD::VECTOR_SPLICE", SDTVecSlice, []>;
703
def build_vector : SDNode<"ISD::BUILD_VECTOR", SDTypeProfile<1, -1, []>, []>;
704
def splat_vector : SDNode<"ISD::SPLAT_VECTOR", SDTypeProfile<1, 1, []>, []>;
705
def step_vector : SDNode<"ISD::STEP_VECTOR", SDTypeProfile<1, 1,
706
                       [SDTCisVec<0>, SDTCisInt<1>]>, []>;
707
def scalar_to_vector : SDNode<"ISD::SCALAR_TO_VECTOR", SDTypeProfile<1, 1, []>,
708
                              []>;
709
 
710
// vector_extract/vector_insert are deprecated. extractelt/insertelt
711
// are preferred.
712
def vector_extract : SDNode<"ISD::EXTRACT_VECTOR_ELT",
713
    SDTypeProfile<1, 2, [SDTCisPtrTy<2>]>, []>;
714
def vector_insert : SDNode<"ISD::INSERT_VECTOR_ELT",
715
    SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisPtrTy<3>]>, []>;
716
def concat_vectors : SDNode<"ISD::CONCAT_VECTORS",
717
    SDTypeProfile<1, 2, [SDTCisSubVecOfVec<1, 0>, SDTCisSameAs<1, 2>]>,[]>;
718
 
719
// This operator does not do subvector type checking.  The ARM
720
// backend, at least, needs it.
721
def vector_extract_subvec : SDNode<"ISD::EXTRACT_SUBVECTOR",
722
    SDTypeProfile<1, 2, [SDTCisInt<2>, SDTCisVec<1>, SDTCisVec<0>]>,
723
    []>;
724
def vector_insert_subvec : SDNode<"ISD::INSERT_SUBVECTOR",
725
    SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVec<2>, SDTCisInt<3>]>,
726
    []>;
727
 
728
// This operator does subvector type checking.
729
def extract_subvector : SDNode<"ISD::EXTRACT_SUBVECTOR", SDTSubVecExtract, []>;
730
def insert_subvector : SDNode<"ISD::INSERT_SUBVECTOR", SDTSubVecInsert, []>;
731
 
732
// Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
733
// these internally.  Don't reference these directly.
734
def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
735
                            SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
736
                            [SDNPHasChain]>;
737
def intrinsic_w_chain : SDNode<"ISD::INTRINSIC_W_CHAIN",
738
                               SDTypeProfile<1, -1, [SDTCisPtrTy<1>]>,
739
                               [SDNPHasChain]>;
740
def intrinsic_wo_chain : SDNode<"ISD::INTRINSIC_WO_CHAIN",
741
                                SDTypeProfile<1, -1, [SDTCisPtrTy<1>]>, []>;
742
 
743
def SDT_assert : SDTypeProfile<1, 1,
744
  [SDTCisInt<0>, SDTCisInt<1>, SDTCisSameAs<1, 0>]>;
745
def assertsext : SDNode<"ISD::AssertSext", SDT_assert>;
746
def assertzext : SDNode<"ISD::AssertZext", SDT_assert>;
747
def assertalign : SDNode<"ISD::AssertAlign", SDT_assert>;
748
 
749
//===----------------------------------------------------------------------===//
750
// Selection DAG Condition Codes
751
 
752
class CondCode<string fcmpName = "", string icmpName = ""> {
753
  string ICmpPredicate = icmpName;
754
  string FCmpPredicate = fcmpName;
755
}
756
 
757
// ISD::CondCode enums, and mapping to CmpInst::Predicate names
758
def SETOEQ : CondCode<"FCMP_OEQ">;
759
def SETOGT : CondCode<"FCMP_OGT">;
760
def SETOGE : CondCode<"FCMP_OGE">;
761
def SETOLT : CondCode<"FCMP_OLT">;
762
def SETOLE : CondCode<"FCMP_OLE">;
763
def SETONE : CondCode<"FCMP_ONE">;
764
def SETO   : CondCode<"FCMP_ORD">;
765
def SETUO  : CondCode<"FCMP_UNO">;
766
def SETUEQ : CondCode<"FCMP_UEQ">;
767
def SETUGT : CondCode<"FCMP_UGT", "ICMP_UGT">;
768
def SETUGE : CondCode<"FCMP_UGE", "ICMP_UGE">;
769
def SETULT : CondCode<"FCMP_ULT", "ICMP_ULT">;
770
def SETULE : CondCode<"FCMP_ULE", "ICMP_ULE">;
771
def SETUNE : CondCode<"FCMP_UNE">;
772
def SETEQ : CondCode<"", "ICMP_EQ">;
773
def SETGT : CondCode<"", "ICMP_SGT">;
774
def SETGE : CondCode<"", "ICMP_SGE">;
775
def SETLT : CondCode<"", "ICMP_SLT">;
776
def SETLE : CondCode<"", "ICMP_SLE">;
777
def SETNE : CondCode<"", "ICMP_NE">;
778
 
779
//===----------------------------------------------------------------------===//
780
// Selection DAG Node Transformation Functions.
781
//
782
// This mechanism allows targets to manipulate nodes in the output DAG once a
783
// match has been formed.  This is typically used to manipulate immediate
784
// values.
785
//
786
class SDNodeXForm<SDNode opc, code xformFunction> {
787
  SDNode Opcode = opc;
788
  code XFormFunction = xformFunction;
789
}
790
 
791
def NOOP_SDNodeXForm : SDNodeXForm<imm, [{}]>;
792
 
793
//===----------------------------------------------------------------------===//
794
// Selection DAG Pattern Fragments.
795
//
796
// Pattern fragments are reusable chunks of dags that match specific things.
797
// They can take arguments and have C++ predicates that control whether they
798
// match.  They are intended to make the patterns for common instructions more
799
// compact and readable.
800
//
801
 
802
/// PatFrags - Represents a set of pattern fragments.  Each single fragment
803
/// can match something on the DAG, from a single node to multiple nested other
804
/// fragments.   The whole set of fragments matches if any of the single
805
/// fragments match.  This allows e.g. matching and "add with overflow" and
806
/// a regular "add" with the same fragment set.
807
///
808
class PatFrags<dag ops, list<dag> frags, code pred = [{}],
809
               SDNodeXForm xform = NOOP_SDNodeXForm> : SDPatternOperator {
810
  dag Operands = ops;
811
  list<dag> Fragments = frags;
812
  code PredicateCode = pred;
813
  code GISelPredicateCode = [{}];
814
  code ImmediateCode = [{}];
815
  SDNodeXForm OperandTransform = xform;
816
 
817
  // When this is set, the PredicateCode may refer to a constant Operands
818
  // vector which contains the captured nodes of the DAG, in the order listed
819
  // by the Operands field above.
820
  //
821
  // This is useful when Fragments involves associative / commutative
822
  // operators: a single piece of code can easily refer to all operands even
823
  // when re-associated / commuted variants of the fragment are matched.
824
  bit PredicateCodeUsesOperands = false;
825
 
826
  // Define a few pre-packaged predicates. This helps GlobalISel import
827
  // existing rules from SelectionDAG for many common cases.
828
  // They will be tested prior to the code in pred and must not be used in
829
  // ImmLeaf and its subclasses.
830
 
831
  // If set to true, a predicate is added that checks for the absence of use of
832
  // the first result.
833
  bit HasNoUse = ?;
834
 
835
  // Is the desired pre-packaged predicate for a load?
836
  bit IsLoad = ?;
837
  // Is the desired pre-packaged predicate for a store?
838
  bit IsStore = ?;
839
  // Is the desired pre-packaged predicate for an atomic?
840
  bit IsAtomic = ?;
841
 
842
  // cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
843
  // cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
844
  bit IsUnindexed = ?;
845
 
846
  // cast<LoadSDNode>(N)->getExtensionType() != ISD::NON_EXTLOAD
847
  bit IsNonExtLoad = ?;
848
  // cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
849
  bit IsAnyExtLoad = ?;
850
  // cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
851
  bit IsSignExtLoad = ?;
852
  // cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
853
  bit IsZeroExtLoad = ?;
854
  // !cast<StoreSDNode>(N)->isTruncatingStore();
855
  // cast<StoreSDNode>(N)->isTruncatingStore();
856
  bit IsTruncStore = ?;
857
 
858
  // cast<MemSDNode>(N)->getAddressSpace() ==
859
  // If this empty, accept any address space.
860
  list<int> AddressSpaces = ?;
861
 
862
  // cast<MemSDNode>(N)->getAlign() >=
863
  // If this is empty, accept any alignment.
864
  int MinAlignment = ?;
865
 
866
  // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Monotonic
867
  bit IsAtomicOrderingMonotonic = ?;
868
  // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Acquire
869
  bit IsAtomicOrderingAcquire = ?;
870
  // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Release
871
  bit IsAtomicOrderingRelease = ?;
872
  // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::AcquireRelease
873
  bit IsAtomicOrderingAcquireRelease = ?;
874
  // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent
875
  bit IsAtomicOrderingSequentiallyConsistent = ?;
876
 
877
  // isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())
878
  // !isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())
879
  bit IsAtomicOrderingAcquireOrStronger = ?;
880
 
881
  // isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())
882
  // !isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())
883
  bit IsAtomicOrderingReleaseOrStronger = ?;
884
 
885
  // cast<LoadSDNode>(N)->getMemoryVT() == MVT::<VT>;
886
  // cast<StoreSDNode>(N)->getMemoryVT() == MVT::<VT>;
887
  ValueType MemoryVT = ?;
888
  // cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>;
889
  // cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>;
890
  ValueType ScalarMemoryVT = ?;
891
}
892
 
893
// PatFrag - A version of PatFrags matching only a single fragment.
894
class PatFrag<dag ops, dag frag, code pred = [{}],
895
              SDNodeXForm xform = NOOP_SDNodeXForm>
896
  : PatFrags<ops, [frag], pred, xform>;
897
 
898
// OutPatFrag is a pattern fragment that is used as part of an output pattern
899
// (not an input pattern). These do not have predicates or transforms, but are
900
// used to avoid repeated subexpressions in output patterns.
901
class OutPatFrag<dag ops, dag frag>
902
 : PatFrag<ops, frag, [{}], NOOP_SDNodeXForm>;
903
 
904
// PatLeaf's are pattern fragments that have no operands.  This is just a helper
905
// to define immediates and other common things concisely.
906
class PatLeaf<dag frag, code pred = [{}], SDNodeXForm xform = NOOP_SDNodeXForm>
907
 : PatFrag<(ops), frag, pred, xform>;
908
 
909
 
910
// ImmLeaf is a pattern fragment with a constraint on the immediate.  The
911
// constraint is a function that is run on the immediate (always with the value
912
// sign extended out to an int64_t) as Imm.  For example:
913
//
914
//  def immSExt8 : ImmLeaf<i16, [{ return (char)Imm == Imm; }]>;
915
//
916
// this is a more convenient form to match 'imm' nodes in than PatLeaf and also
917
// is preferred over using PatLeaf because it allows the code generator to
918
// reason more about the constraint.
919
//
920
// If FastIsel should ignore all instructions that have an operand of this type,
921
// the FastIselShouldIgnore flag can be set.  This is an optimization to reduce
922
// the code size of the generated fast instruction selector.
923
class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
924
              SDNode ImmNode = imm>
925
  : PatFrag<(ops), (vt ImmNode), [{}], xform> {
926
  let ImmediateCode = pred;
927
  bit FastIselShouldIgnore = false;
928
 
929
  // Is the data type of the immediate an APInt?
930
  bit IsAPInt = false;
931
 
932
  // Is the data type of the immediate an APFloat?
933
  bit IsAPFloat = false;
934
}
935
 
936
// Convenience wrapper for ImmLeaf to use timm/TargetConstant instead
937
// of imm/Constant.
938
class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
939
  SDNode ImmNode = timm> : ImmLeaf<vt, pred, xform, ImmNode>;
940
 
941
// An ImmLeaf except that Imm is an APInt. This is useful when you need to
942
// zero-extend the immediate instead of sign-extend it.
943
//
944
// Note that FastISel does not currently understand IntImmLeaf and will not
945
// generate code for rules that make use of it. As such, it does not make sense
946
// to replace ImmLeaf with IntImmLeaf. However, replacing PatLeaf with an
947
// IntImmLeaf will allow GlobalISel to import the rule.
948
class IntImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm>
949
    : ImmLeaf<vt, pred, xform> {
950
  let IsAPInt = true;
951
  let FastIselShouldIgnore = true;
952
}
953
 
954
// An ImmLeaf except that Imm is an APFloat.
955
//
956
// Note that FastISel does not currently understand FPImmLeaf and will not
957
// generate code for rules that make use of it.
958
class FPImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm>
959
  : ImmLeaf<vt, pred, xform, fpimm> {
960
  let IsAPFloat = true;
961
  let FastIselShouldIgnore = true;
962
}
963
 
964
// Leaf fragments.
965
 
966
def vtInt      : PatLeaf<(vt),  [{ return N->getVT().isInteger(); }]>;
967
def vtFP       : PatLeaf<(vt),  [{ return N->getVT().isFloatingPoint(); }]>;
968
 
969
// Use ISD::isConstantSplatVectorAllOnes or ISD::isConstantSplatVectorAllZeros
970
// to look for the corresponding build_vector or splat_vector. Will look through
971
// bitcasts and check for either opcode, except when used as a pattern root.
972
// When used as a pattern root, only fixed-length build_vector and scalable
973
// splat_vector are supported.
974
def immAllOnesV  : SDPatternOperator; // ISD::isConstantSplatVectorAllOnes
975
def immAllZerosV : SDPatternOperator; // ISD::isConstantSplatVectorAllZeros
976
 
977
// Other helper fragments.
978
def not  : PatFrag<(ops node:$in), (xor node:$in, -1)>;
979
def vnot : PatFrag<(ops node:$in), (xor node:$in, immAllOnesV)>;
980
def ineg : PatFrag<(ops node:$in), (sub 0, node:$in)>;
981
 
982
def zanyext : PatFrags<(ops node:$op),
983
                       [(zext node:$op),
984
                        (anyext node:$op)]>;
985
 
986
// null_frag - The null pattern operator is used in multiclass instantiations
987
// which accept an SDPatternOperator for use in matching patterns for internal
988
// definitions. When expanding a pattern, if the null fragment is referenced
989
// in the expansion, the pattern is discarded and it is as-if '[]' had been
990
// specified. This allows multiclasses to have the isel patterns be optional.
991
def null_frag : SDPatternOperator;
992
 
993
// load fragments.
994
def unindexedload : PatFrag<(ops node:$ptr), (ld node:$ptr)> {
995
  let IsLoad = true;
996
  let IsUnindexed = true;
997
}
998
def load : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
999
  let IsLoad = true;
1000
  let IsNonExtLoad = true;
1001
}
1002
 
1003
// extending load fragments.
1004
def extload   : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
1005
  let IsLoad = true;
1006
  let IsAnyExtLoad = true;
1007
}
1008
def sextload  : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
1009
  let IsLoad = true;
1010
  let IsSignExtLoad = true;
1011
}
1012
def zextload  : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
1013
  let IsLoad = true;
1014
  let IsZeroExtLoad = true;
1015
}
1016
 
1017
def extloadi1  : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1018
  let IsLoad = true;
1019
  let MemoryVT = i1;
1020
}
1021
def extloadi8  : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1022
  let IsLoad = true;
1023
  let MemoryVT = i8;
1024
}
1025
def extloadi16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1026
  let IsLoad = true;
1027
  let MemoryVT = i16;
1028
}
1029
def extloadi32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1030
  let IsLoad = true;
1031
  let MemoryVT = i32;
1032
}
1033
def extloadf16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1034
  let IsLoad = true;
1035
  let MemoryVT = f16;
1036
}
1037
def extloadf32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1038
  let IsLoad = true;
1039
  let MemoryVT = f32;
1040
}
1041
def extloadf64 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1042
  let IsLoad = true;
1043
  let MemoryVT = f64;
1044
}
1045
 
1046
def sextloadi1  : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1047
  let IsLoad = true;
1048
  let MemoryVT = i1;
1049
}
1050
def sextloadi8  : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1051
  let IsLoad = true;
1052
  let MemoryVT = i8;
1053
}
1054
def sextloadi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1055
  let IsLoad = true;
1056
  let MemoryVT = i16;
1057
}
1058
def sextloadi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1059
  let IsLoad = true;
1060
  let MemoryVT = i32;
1061
}
1062
 
1063
def zextloadi1  : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1064
  let IsLoad = true;
1065
  let MemoryVT = i1;
1066
}
1067
def zextloadi8  : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1068
  let IsLoad = true;
1069
  let MemoryVT = i8;
1070
}
1071
def zextloadi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1072
  let IsLoad = true;
1073
  let MemoryVT = i16;
1074
}
1075
def zextloadi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1076
  let IsLoad = true;
1077
  let MemoryVT = i32;
1078
}
1079
 
1080
def extloadvi1  : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1081
  let IsLoad = true;
1082
  let ScalarMemoryVT = i1;
1083
}
1084
def extloadvi8  : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1085
  let IsLoad = true;
1086
  let ScalarMemoryVT = i8;
1087
}
1088
def extloadvi16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1089
  let IsLoad = true;
1090
  let ScalarMemoryVT = i16;
1091
}
1092
def extloadvi32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1093
  let IsLoad = true;
1094
  let ScalarMemoryVT = i32;
1095
}
1096
def extloadvf16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1097
  let IsLoad = true;
1098
  let ScalarMemoryVT = f16;
1099
}
1100
def extloadvf32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1101
  let IsLoad = true;
1102
  let ScalarMemoryVT = f32;
1103
}
1104
def extloadvf64 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1105
  let IsLoad = true;
1106
  let ScalarMemoryVT = f64;
1107
}
1108
 
1109
def sextloadvi1  : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1110
  let IsLoad = true;
1111
  let ScalarMemoryVT = i1;
1112
}
1113
def sextloadvi8  : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1114
  let IsLoad = true;
1115
  let ScalarMemoryVT = i8;
1116
}
1117
def sextloadvi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1118
  let IsLoad = true;
1119
  let ScalarMemoryVT = i16;
1120
}
1121
def sextloadvi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
1122
  let IsLoad = true;
1123
  let ScalarMemoryVT = i32;
1124
}
1125
 
1126
def zextloadvi1  : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1127
  let IsLoad = true;
1128
  let ScalarMemoryVT = i1;
1129
}
1130
def zextloadvi8  : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1131
  let IsLoad = true;
1132
  let ScalarMemoryVT = i8;
1133
}
1134
def zextloadvi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1135
  let IsLoad = true;
1136
  let ScalarMemoryVT = i16;
1137
}
1138
def zextloadvi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
1139
  let IsLoad = true;
1140
  let ScalarMemoryVT = i32;
1141
}
1142
 
1143
// store fragments.
1144
def unindexedstore : PatFrag<(ops node:$val, node:$ptr),
1145
                             (st node:$val, node:$ptr)> {
1146
  let IsStore = true;
1147
  let IsUnindexed = true;
1148
}
1149
def store : PatFrag<(ops node:$val, node:$ptr),
1150
                    (unindexedstore node:$val, node:$ptr)> {
1151
  let IsStore = true;
1152
  let IsTruncStore = false;
1153
}
1154
 
1155
// truncstore fragments.
1156
def truncstore : PatFrag<(ops node:$val, node:$ptr),
1157
                         (unindexedstore node:$val, node:$ptr)> {
1158
  let IsStore = true;
1159
  let IsTruncStore = true;
1160
}
1161
def truncstorei8 : PatFrag<(ops node:$val, node:$ptr),
1162
                           (truncstore node:$val, node:$ptr)> {
1163
  let IsStore = true;
1164
  let MemoryVT = i8;
1165
  let IsTruncStore = true;
1166
}
1167
def truncstorei16 : PatFrag<(ops node:$val, node:$ptr),
1168
                            (truncstore node:$val, node:$ptr)> {
1169
  let IsStore = true;
1170
  let MemoryVT = i16;
1171
  let IsTruncStore = true;
1172
}
1173
def truncstorei32 : PatFrag<(ops node:$val, node:$ptr),
1174
                            (truncstore node:$val, node:$ptr)> {
1175
  let IsStore = true;
1176
  let MemoryVT = i32;
1177
  let IsTruncStore = true;
1178
}
1179
def truncstoref16 : PatFrag<(ops node:$val, node:$ptr),
1180
                            (truncstore node:$val, node:$ptr)> {
1181
  let IsStore = true;
1182
  let MemoryVT = f16;
1183
}
1184
def truncstoref32 : PatFrag<(ops node:$val, node:$ptr),
1185
                            (truncstore node:$val, node:$ptr)> {
1186
  let IsStore = true;
1187
  let MemoryVT = f32;
1188
}
1189
def truncstoref64 : PatFrag<(ops node:$val, node:$ptr),
1190
                            (truncstore node:$val, node:$ptr)> {
1191
  let IsStore = true;
1192
  let MemoryVT = f64;
1193
}
1194
 
1195
def truncstorevi8 : PatFrag<(ops node:$val, node:$ptr),
1196
                            (truncstore node:$val, node:$ptr)> {
1197
  let IsStore = true;
1198
  let ScalarMemoryVT = i8;
1199
}
1200
 
1201
def truncstorevi16 : PatFrag<(ops node:$val, node:$ptr),
1202
                             (truncstore node:$val, node:$ptr)> {
1203
  let IsStore = true;
1204
  let ScalarMemoryVT = i16;
1205
}
1206
 
1207
def truncstorevi32 : PatFrag<(ops node:$val, node:$ptr),
1208
                             (truncstore node:$val, node:$ptr)> {
1209
  let IsStore = true;
1210
  let ScalarMemoryVT = i32;
1211
}
1212
 
1213
// indexed store fragments.
1214
def istore : PatFrag<(ops node:$val, node:$base, node:$offset),
1215
                     (ist node:$val, node:$base, node:$offset)> {
1216
  let IsStore = true;
1217
  let IsTruncStore = false;
1218
}
1219
 
1220
def pre_store : PatFrag<(ops node:$val, node:$base, node:$offset),
1221
                        (istore node:$val, node:$base, node:$offset), [{
1222
  ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
1223
  return AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1224
}]>;
1225
 
1226
def itruncstore : PatFrag<(ops node:$val, node:$base, node:$offset),
1227
                          (ist node:$val, node:$base, node:$offset)> {
1228
  let IsStore = true;
1229
  let IsTruncStore = true;
1230
}
1231
def pre_truncst : PatFrag<(ops node:$val, node:$base, node:$offset),
1232
                          (itruncstore node:$val, node:$base, node:$offset), [{
1233
  ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
1234
  return AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1235
}]>;
1236
def pre_truncsti1 : PatFrag<(ops node:$val, node:$base, node:$offset),
1237
                            (pre_truncst node:$val, node:$base, node:$offset)> {
1238
  let IsStore = true;
1239
  let MemoryVT = i1;
1240
}
1241
def pre_truncsti8 : PatFrag<(ops node:$val, node:$base, node:$offset),
1242
                            (pre_truncst node:$val, node:$base, node:$offset)> {
1243
  let IsStore = true;
1244
  let MemoryVT = i8;
1245
}
1246
def pre_truncsti16 : PatFrag<(ops node:$val, node:$base, node:$offset),
1247
                             (pre_truncst node:$val, node:$base, node:$offset)> {
1248
  let IsStore = true;
1249
  let MemoryVT = i16;
1250
}
1251
def pre_truncsti32 : PatFrag<(ops node:$val, node:$base, node:$offset),
1252
                             (pre_truncst node:$val, node:$base, node:$offset)> {
1253
  let IsStore = true;
1254
  let MemoryVT = i32;
1255
}
1256
def pre_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset),
1257
                             (pre_truncst node:$val, node:$base, node:$offset)> {
1258
  let IsStore = true;
1259
  let MemoryVT = f32;
1260
}
1261
def pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
1262
                             (pre_truncst node:$val, node:$base, node:$offset)> {
1263
  let IsStore = true;
1264
  let ScalarMemoryVT = i8;
1265
}
1266
def pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
1267
                              (pre_truncst node:$val, node:$base, node:$offset)> {
1268
  let IsStore = true;
1269
  let ScalarMemoryVT = i16;
1270
}
1271
 
1272
def post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
1273
                         (istore node:$val, node:$ptr, node:$offset), [{
1274
  ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
1275
  return AM == ISD::POST_INC || AM == ISD::POST_DEC;
1276
}]>;
1277
 
1278
def post_truncst : PatFrag<(ops node:$val, node:$base, node:$offset),
1279
                           (itruncstore node:$val, node:$base, node:$offset), [{
1280
  ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode();
1281
  return AM == ISD::POST_INC || AM == ISD::POST_DEC;
1282
}]>;
1283
def post_truncsti1 : PatFrag<(ops node:$val, node:$base, node:$offset),
1284
                             (post_truncst node:$val, node:$base, node:$offset)> {
1285
  let IsStore = true;
1286
  let MemoryVT = i1;
1287
}
1288
def post_truncsti8 : PatFrag<(ops node:$val, node:$base, node:$offset),
1289
                             (post_truncst node:$val, node:$base, node:$offset)> {
1290
  let IsStore = true;
1291
  let MemoryVT = i8;
1292
}
1293
def post_truncsti16 : PatFrag<(ops node:$val, node:$base, node:$offset),
1294
                              (post_truncst node:$val, node:$base, node:$offset)> {
1295
  let IsStore = true;
1296
  let MemoryVT = i16;
1297
}
1298
def post_truncsti32 : PatFrag<(ops node:$val, node:$base, node:$offset),
1299
                              (post_truncst node:$val, node:$base, node:$offset)> {
1300
  let IsStore = true;
1301
  let MemoryVT = i32;
1302
}
1303
def post_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset),
1304
                              (post_truncst node:$val, node:$base, node:$offset)> {
1305
  let IsStore = true;
1306
  let MemoryVT = f32;
1307
}
1308
def post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
1309
                              (post_truncst node:$val, node:$base, node:$offset)> {
1310
  let IsStore = true;
1311
  let ScalarMemoryVT = i8;
1312
}
1313
def post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
1314
                               (post_truncst node:$val, node:$base, node:$offset)> {
1315
  let IsStore = true;
1316
  let ScalarMemoryVT = i16;
1317
}
1318
 
1319
// A helper for matching undef or freeze undef
1320
def undef_or_freeze_undef : PatFrags<(ops), [(undef), (freeze undef)]>;
1321
 
1322
// TODO: Split these into volatile and unordered flavors to enable
1323
// selectively legal optimizations for each.  (See D66309)
1324
def simple_load : PatFrag<(ops node:$ptr),
1325
                          (load node:$ptr), [{
1326
  return cast<LoadSDNode>(N)->isSimple();
1327
}]>;
1328
def simple_store : PatFrag<(ops node:$val, node:$ptr),
1329
                           (store node:$val, node:$ptr), [{
1330
  return cast<StoreSDNode>(N)->isSimple();
1331
}]>;
1332
 
1333
// nontemporal store fragments.
1334
def nontemporalstore : PatFrag<(ops node:$val, node:$ptr),
1335
                               (store node:$val, node:$ptr), [{
1336
  return cast<StoreSDNode>(N)->isNonTemporal();
1337
}]>;
1338
 
1339
def alignednontemporalstore : PatFrag<(ops node:$val, node:$ptr),
1340
                                      (nontemporalstore node:$val, node:$ptr), [{
1341
  StoreSDNode *St = cast<StoreSDNode>(N);
1342
  return St->getAlign() >= St->getMemoryVT().getStoreSize();
1343
}]>;
1344
 
1345
def unalignednontemporalstore : PatFrag<(ops node:$val, node:$ptr),
1346
                                        (nontemporalstore node:$val, node:$ptr), [{
1347
  StoreSDNode *St = cast<StoreSDNode>(N);
1348
  return St->getAlignment() < St->getMemoryVT().getStoreSize();
1349
}]>;
1350
 
1351
// nontemporal load fragments.
1352
def nontemporalload : PatFrag<(ops node:$ptr),
1353
                               (load node:$ptr), [{
1354
  return cast<LoadSDNode>(N)->isNonTemporal();
1355
}]>;
1356
 
1357
def alignednontemporalload : PatFrag<(ops node:$ptr),
1358
                                      (nontemporalload node:$ptr), [{
1359
  LoadSDNode *Ld = cast<LoadSDNode>(N);
1360
  return Ld->getAlign() >= Ld->getMemoryVT().getStoreSize();
1361
}]>;
1362
 
1363
// setcc convenience fragments.
1364
def setoeq : PatFrag<(ops node:$lhs, node:$rhs),
1365
                     (setcc node:$lhs, node:$rhs, SETOEQ)>;
1366
def setogt : PatFrag<(ops node:$lhs, node:$rhs),
1367
                     (setcc node:$lhs, node:$rhs, SETOGT)>;
1368
def setoge : PatFrag<(ops node:$lhs, node:$rhs),
1369
                     (setcc node:$lhs, node:$rhs, SETOGE)>;
1370
def setolt : PatFrag<(ops node:$lhs, node:$rhs),
1371
                     (setcc node:$lhs, node:$rhs, SETOLT)>;
1372
def setole : PatFrag<(ops node:$lhs, node:$rhs),
1373
                     (setcc node:$lhs, node:$rhs, SETOLE)>;
1374
def setone : PatFrag<(ops node:$lhs, node:$rhs),
1375
                     (setcc node:$lhs, node:$rhs, SETONE)>;
1376
def seto   : PatFrag<(ops node:$lhs, node:$rhs),
1377
                     (setcc node:$lhs, node:$rhs, SETO)>;
1378
def setuo  : PatFrag<(ops node:$lhs, node:$rhs),
1379
                     (setcc node:$lhs, node:$rhs, SETUO)>;
1380
def setueq : PatFrag<(ops node:$lhs, node:$rhs),
1381
                     (setcc node:$lhs, node:$rhs, SETUEQ)>;
1382
def setugt : PatFrag<(ops node:$lhs, node:$rhs),
1383
                     (setcc node:$lhs, node:$rhs, SETUGT)>;
1384
def setuge : PatFrag<(ops node:$lhs, node:$rhs),
1385
                     (setcc node:$lhs, node:$rhs, SETUGE)>;
1386
def setult : PatFrag<(ops node:$lhs, node:$rhs),
1387
                     (setcc node:$lhs, node:$rhs, SETULT)>;
1388
def setule : PatFrag<(ops node:$lhs, node:$rhs),
1389
                     (setcc node:$lhs, node:$rhs, SETULE)>;
1390
def setune : PatFrag<(ops node:$lhs, node:$rhs),
1391
                     (setcc node:$lhs, node:$rhs, SETUNE)>;
1392
def seteq  : PatFrag<(ops node:$lhs, node:$rhs),
1393
                     (setcc node:$lhs, node:$rhs, SETEQ)>;
1394
def setgt  : PatFrag<(ops node:$lhs, node:$rhs),
1395
                     (setcc node:$lhs, node:$rhs, SETGT)>;
1396
def setge  : PatFrag<(ops node:$lhs, node:$rhs),
1397
                     (setcc node:$lhs, node:$rhs, SETGE)>;
1398
def setlt  : PatFrag<(ops node:$lhs, node:$rhs),
1399
                     (setcc node:$lhs, node:$rhs, SETLT)>;
1400
def setle  : PatFrag<(ops node:$lhs, node:$rhs),
1401
                     (setcc node:$lhs, node:$rhs, SETLE)>;
1402
def setne  : PatFrag<(ops node:$lhs, node:$rhs),
1403
                     (setcc node:$lhs, node:$rhs, SETNE)>;
1404
 
1405
// We don't have strict FP extended loads as single DAG nodes, but we can
1406
// still provide convenience fragments to match those operations.
1407
def strict_extloadf32 : PatFrag<(ops node:$ptr),
1408
                                (strict_fpextend (f32 (load node:$ptr)))>;
1409
def strict_extloadf64 : PatFrag<(ops node:$ptr),
1410
                                (strict_fpextend (f64 (load node:$ptr)))>;
1411
 
1412
// Convenience fragments to match both strict and non-strict fp operations
1413
def any_fadd       : PatFrags<(ops node:$lhs, node:$rhs),
1414
                              [(strict_fadd node:$lhs, node:$rhs),
1415
                               (fadd node:$lhs, node:$rhs)]>;
1416
def any_fsub       : PatFrags<(ops node:$lhs, node:$rhs),
1417
                              [(strict_fsub node:$lhs, node:$rhs),
1418
                               (fsub node:$lhs, node:$rhs)]>;
1419
def any_fmul       : PatFrags<(ops node:$lhs, node:$rhs),
1420
                              [(strict_fmul node:$lhs, node:$rhs),
1421
                               (fmul node:$lhs, node:$rhs)]>;
1422
def any_fdiv       : PatFrags<(ops node:$lhs, node:$rhs),
1423
                              [(strict_fdiv node:$lhs, node:$rhs),
1424
                               (fdiv node:$lhs, node:$rhs)]>;
1425
def any_frem       : PatFrags<(ops node:$lhs, node:$rhs),
1426
                              [(strict_frem node:$lhs, node:$rhs),
1427
                               (frem node:$lhs, node:$rhs)]>;
1428
def any_fma        : PatFrags<(ops node:$src1, node:$src2, node:$src3),
1429
                              [(strict_fma node:$src1, node:$src2, node:$src3),
1430
                               (fma node:$src1, node:$src2, node:$src3)]>;
1431
def any_fsqrt      : PatFrags<(ops node:$src),
1432
                              [(strict_fsqrt node:$src),
1433
                               (fsqrt node:$src)]>;
1434
def any_fsin       : PatFrags<(ops node:$src),
1435
                              [(strict_fsin node:$src),
1436
                               (fsin node:$src)]>;
1437
def any_fcos       : PatFrags<(ops node:$src),
1438
                              [(strict_fcos node:$src),
1439
                               (fcos node:$src)]>;
1440
def any_fexp2      : PatFrags<(ops node:$src),
1441
                              [(strict_fexp2 node:$src),
1442
                               (fexp2 node:$src)]>;
1443
def any_fpow       : PatFrags<(ops node:$lhs, node:$rhs),
1444
                              [(strict_fpow node:$lhs, node:$rhs),
1445
                               (fpow node:$lhs, node:$rhs)]>;
1446
def any_flog2      : PatFrags<(ops node:$src),
1447
                              [(strict_flog2 node:$src),
1448
                               (flog2 node:$src)]>;
1449
def any_frint      : PatFrags<(ops node:$src),
1450
                              [(strict_frint node:$src),
1451
                               (frint node:$src)]>;
1452
def any_lrint      : PatFrags<(ops node:$src),
1453
                              [(strict_lrint node:$src),
1454
                               (lrint node:$src)]>;
1455
def any_llrint     : PatFrags<(ops node:$src),
1456
                              [(strict_llrint node:$src),
1457
                               (llrint node:$src)]>;
1458
def any_fnearbyint : PatFrags<(ops node:$src),
1459
                              [(strict_fnearbyint node:$src),
1460
                               (fnearbyint node:$src)]>;
1461
def any_fceil      : PatFrags<(ops node:$src),
1462
                              [(strict_fceil node:$src),
1463
                               (fceil node:$src)]>;
1464
def any_ffloor     : PatFrags<(ops node:$src),
1465
                              [(strict_ffloor node:$src),
1466
                               (ffloor node:$src)]>;
1467
def any_lround     : PatFrags<(ops node:$src),
1468
                              [(strict_lround node:$src),
1469
                               (lround node:$src)]>;
1470
def any_llround    : PatFrags<(ops node:$src),
1471
                              [(strict_llround node:$src),
1472
                               (llround node:$src)]>;
1473
def any_fround     : PatFrags<(ops node:$src),
1474
                              [(strict_fround node:$src),
1475
                               (fround node:$src)]>;
1476
def any_froundeven : PatFrags<(ops node:$src),
1477
                              [(strict_froundeven node:$src),
1478
                               (froundeven node:$src)]>;
1479
def any_ftrunc     : PatFrags<(ops node:$src),
1480
                              [(strict_ftrunc node:$src),
1481
                               (ftrunc node:$src)]>;
1482
def any_fmaxnum    : PatFrags<(ops node:$lhs, node:$rhs),
1483
                              [(strict_fmaxnum node:$lhs, node:$rhs),
1484
                               (fmaxnum node:$lhs, node:$rhs)]>;
1485
def any_fminnum    : PatFrags<(ops node:$lhs, node:$rhs),
1486
                              [(strict_fminnum node:$lhs, node:$rhs),
1487
                               (fminnum node:$lhs, node:$rhs)]>;
1488
def any_fmaximum   : PatFrags<(ops node:$lhs, node:$rhs),
1489
                              [(strict_fmaximum node:$lhs, node:$rhs),
1490
                               (fmaximum node:$lhs, node:$rhs)]>;
1491
def any_fminimum   : PatFrags<(ops node:$lhs, node:$rhs),
1492
                              [(strict_fminimum node:$lhs, node:$rhs),
1493
                               (fminimum node:$lhs, node:$rhs)]>;
1494
def any_fpround    : PatFrags<(ops node:$src),
1495
                              [(strict_fpround node:$src),
1496
                               (fpround node:$src)]>;
1497
def any_fpextend   : PatFrags<(ops node:$src),
1498
                              [(strict_fpextend node:$src),
1499
                               (fpextend node:$src)]>;
1500
def any_extloadf32 : PatFrags<(ops node:$ptr),
1501
                              [(strict_extloadf32 node:$ptr),
1502
                               (extloadf32 node:$ptr)]>;
1503
def any_extloadf64 : PatFrags<(ops node:$ptr),
1504
                              [(strict_extloadf64 node:$ptr),
1505
                               (extloadf64 node:$ptr)]>;
1506
def any_fp_to_sint : PatFrags<(ops node:$src),
1507
                              [(strict_fp_to_sint node:$src),
1508
                               (fp_to_sint node:$src)]>;
1509
def any_fp_to_uint : PatFrags<(ops node:$src),
1510
                              [(strict_fp_to_uint node:$src),
1511
                               (fp_to_uint node:$src)]>;
1512
def any_sint_to_fp : PatFrags<(ops node:$src),
1513
                              [(strict_sint_to_fp node:$src),
1514
                               (sint_to_fp node:$src)]>;
1515
def any_uint_to_fp : PatFrags<(ops node:$src),
1516
                              [(strict_uint_to_fp node:$src),
1517
                               (uint_to_fp node:$src)]>;
1518
def any_fsetcc : PatFrags<(ops node:$lhs, node:$rhs, node:$pred),
1519
                          [(strict_fsetcc node:$lhs, node:$rhs, node:$pred),
1520
                           (setcc node:$lhs, node:$rhs, node:$pred)]>;
1521
def any_fsetccs : PatFrags<(ops node:$lhs, node:$rhs, node:$pred),
1522
                          [(strict_fsetccs node:$lhs, node:$rhs, node:$pred),
1523
                           (setcc node:$lhs, node:$rhs, node:$pred)]>;
1524
 
1525
multiclass binary_atomic_op_ord {
1526
  def NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val),
1527
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
1528
    let IsAtomic = true;
1529
    let IsAtomicOrderingMonotonic = true;
1530
  }
1531
  def NAME#_acquire : PatFrag<(ops node:$ptr, node:$val),
1532
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
1533
    let IsAtomic = true;
1534
    let IsAtomicOrderingAcquire = true;
1535
  }
1536
  def NAME#_release : PatFrag<(ops node:$ptr, node:$val),
1537
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
1538
    let IsAtomic = true;
1539
    let IsAtomicOrderingRelease = true;
1540
  }
1541
  def NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$val),
1542
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
1543
    let IsAtomic = true;
1544
    let IsAtomicOrderingAcquireRelease = true;
1545
  }
1546
  def NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$val),
1547
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
1548
    let IsAtomic = true;
1549
    let IsAtomicOrderingSequentiallyConsistent = true;
1550
  }
1551
}
1552
 
1553
multiclass ternary_atomic_op_ord {
1554
  def NAME#_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1555
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val)> {
1556
    let IsAtomic = true;
1557
    let IsAtomicOrderingMonotonic = true;
1558
  }
1559
  def NAME#_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1560
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val)> {
1561
    let IsAtomic = true;
1562
    let IsAtomicOrderingAcquire = true;
1563
  }
1564
  def NAME#_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1565
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val)> {
1566
    let IsAtomic = true;
1567
    let IsAtomicOrderingRelease = true;
1568
  }
1569
  def NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1570
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val)> {
1571
    let IsAtomic = true;
1572
    let IsAtomicOrderingAcquireRelease = true;
1573
  }
1574
  def NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1575
      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val)> {
1576
    let IsAtomic = true;
1577
    let IsAtomicOrderingSequentiallyConsistent = true;
1578
  }
1579
}
1580
 
1581
multiclass binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
1582
  def _8 : PatFrag<(ops node:$ptr, node:$val),
1583
                   (atomic_op  node:$ptr, node:$val)> {
1584
    let IsAtomic = true;
1585
    let MemoryVT = !if(IsInt, i8, ?);
1586
  }
1587
  def _16 : PatFrag<(ops node:$ptr, node:$val),
1588
                    (atomic_op node:$ptr, node:$val)> {
1589
    let IsAtomic = true;
1590
    let MemoryVT = !if(IsInt, i16, f16);
1591
  }
1592
  def _32 : PatFrag<(ops node:$ptr, node:$val),
1593
                    (atomic_op node:$ptr, node:$val)> {
1594
    let IsAtomic = true;
1595
    let MemoryVT = !if(IsInt, i32, f32);
1596
  }
1597
  def _64 : PatFrag<(ops node:$ptr, node:$val),
1598
                    (atomic_op node:$ptr, node:$val)> {
1599
    let IsAtomic = true;
1600
    let MemoryVT = !if(IsInt, i64, f64);
1601
  }
1602
 
1603
  defm NAME#_8  : binary_atomic_op_ord;
1604
  defm NAME#_16 : binary_atomic_op_ord;
1605
  defm NAME#_32 : binary_atomic_op_ord;
1606
  defm NAME#_64 : binary_atomic_op_ord;
1607
}
1608
 
1609
multiclass ternary_atomic_op<SDNode atomic_op> {
1610
  def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1611
                   (atomic_op  node:$ptr, node:$cmp, node:$val)> {
1612
    let IsAtomic = true;
1613
    let MemoryVT = i8;
1614
  }
1615
  def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1616
                    (atomic_op node:$ptr, node:$cmp, node:$val)> {
1617
    let IsAtomic = true;
1618
    let MemoryVT = i16;
1619
  }
1620
  def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1621
                    (atomic_op node:$ptr, node:$cmp, node:$val)> {
1622
    let IsAtomic = true;
1623
    let MemoryVT = i32;
1624
  }
1625
  def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1626
                    (atomic_op node:$ptr, node:$cmp, node:$val)> {
1627
    let IsAtomic = true;
1628
    let MemoryVT = i64;
1629
  }
1630
 
1631
  defm NAME#_8  : ternary_atomic_op_ord;
1632
  defm NAME#_16 : ternary_atomic_op_ord;
1633
  defm NAME#_32 : ternary_atomic_op_ord;
1634
  defm NAME#_64 : ternary_atomic_op_ord;
1635
}
1636
 
1637
defm atomic_load_add  : binary_atomic_op<atomic_load_add>;
1638
defm atomic_swap      : binary_atomic_op<atomic_swap>;
1639
defm atomic_load_sub  : binary_atomic_op<atomic_load_sub>;
1640
defm atomic_load_and  : binary_atomic_op<atomic_load_and>;
1641
defm atomic_load_clr  : binary_atomic_op<atomic_load_clr>;
1642
defm atomic_load_or   : binary_atomic_op<atomic_load_or>;
1643
defm atomic_load_xor  : binary_atomic_op<atomic_load_xor>;
1644
defm atomic_load_nand : binary_atomic_op<atomic_load_nand>;
1645
defm atomic_load_min  : binary_atomic_op<atomic_load_min>;
1646
defm atomic_load_max  : binary_atomic_op<atomic_load_max>;
1647
defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
1648
defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
1649
defm atomic_store     : binary_atomic_op<atomic_store>;
1650
defm atomic_cmp_swap  : ternary_atomic_op<atomic_cmp_swap>;
1651
 
1652
/// Atomic load which zeroes the excess high bits.
1653
def atomic_load_zext :
1654
  PatFrag<(ops node:$ptr), (atomic_load node:$ptr)> {
1655
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1656
  let IsZeroExtLoad = true;
1657
}
1658
 
1659
/// Atomic load which sign extends the excess high bits.
1660
def atomic_load_sext :
1661
  PatFrag<(ops node:$ptr), (atomic_load node:$ptr)> {
1662
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1663
  let IsSignExtLoad = true;
1664
}
1665
 
1666
def atomic_load_8 :
1667
  PatFrag<(ops node:$ptr),
1668
          (atomic_load node:$ptr)> {
1669
  let IsAtomic = true;
1670
  let MemoryVT = i8;
1671
}
1672
 
1673
def atomic_load_16 :
1674
  PatFrag<(ops node:$ptr),
1675
          (atomic_load node:$ptr)> {
1676
  let IsAtomic = true;
1677
  let MemoryVT = i16;
1678
}
1679
 
1680
def atomic_load_32 :
1681
  PatFrag<(ops node:$ptr),
1682
          (atomic_load node:$ptr)> {
1683
  let IsAtomic = true;
1684
  let MemoryVT = i32;
1685
}
1686
def atomic_load_64 :
1687
  PatFrag<(ops node:$ptr),
1688
          (atomic_load node:$ptr)> {
1689
  let IsAtomic = true;
1690
  let MemoryVT = i64;
1691
}
1692
 
1693
def atomic_load_zext_8 :
1694
  PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
1695
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1696
  let MemoryVT = i8;
1697
}
1698
 
1699
def atomic_load_zext_16 :
1700
  PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
1701
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1702
  let MemoryVT = i16;
1703
}
1704
 
1705
def atomic_load_sext_8 :
1706
  PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
1707
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1708
  let MemoryVT = i8;
1709
}
1710
 
1711
def atomic_load_sext_16 :
1712
  PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
1713
  let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1714
  let MemoryVT = i16;
1715
}
1716
 
1717
// Atomic load which zeroes or anyextends the high bits.
1718
def atomic_load_az_8 : PatFrags<(ops node:$op),
1719
                                [(atomic_load_8 node:$op),
1720
                                 (atomic_load_zext_8 node:$op)]>;
1721
 
1722
// Atomic load which zeroes or anyextends the high bits.
1723
def atomic_load_az_16 : PatFrags<(ops node:$op),
1724
                                 [(atomic_load_16 node:$op),
1725
                                  (atomic_load_zext_16 node:$op)]>;
1726
 
1727
def nonext_masked_gather :
1728
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1729
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1730
  return cast<MaskedGatherSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
1731
}]>;
1732
 
1733
// Any extending masked gather fragments.
1734
def ext_masked_gather_i8 :
1735
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1736
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1737
  auto MGN = cast<MaskedGatherSDNode>(N);
1738
  return MGN->getExtensionType() == ISD::EXTLOAD &&
1739
         MGN->getMemoryVT().getScalarType() == MVT::i8;
1740
}]>;
1741
def ext_masked_gather_i16 :
1742
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1743
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1744
  auto MGN = cast<MaskedGatherSDNode>(N);
1745
  return MGN->getExtensionType() == ISD::EXTLOAD &&
1746
         MGN->getMemoryVT().getScalarType() == MVT::i16;
1747
}]>;
1748
def ext_masked_gather_i32 :
1749
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1750
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1751
  auto MGN = cast<MaskedGatherSDNode>(N);
1752
  return MGN->getExtensionType() == ISD::EXTLOAD &&
1753
         MGN->getMemoryVT().getScalarType() == MVT::i32;
1754
}]>;
1755
 
1756
// Sign extending masked gather fragments.
1757
def sext_masked_gather_i8 :
1758
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1759
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1760
  auto MGN = cast<MaskedGatherSDNode>(N);
1761
  return MGN->getExtensionType() == ISD::SEXTLOAD &&
1762
         MGN->getMemoryVT().getScalarType() == MVT::i8;
1763
}]>;
1764
def sext_masked_gather_i16 :
1765
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1766
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1767
  auto MGN = cast<MaskedGatherSDNode>(N);
1768
  return MGN->getExtensionType() == ISD::SEXTLOAD &&
1769
         MGN->getMemoryVT().getScalarType() == MVT::i16;
1770
}]>;
1771
def sext_masked_gather_i32 :
1772
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1773
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1774
  auto MGN = cast<MaskedGatherSDNode>(N);
1775
  return MGN->getExtensionType() == ISD::SEXTLOAD &&
1776
         MGN->getMemoryVT().getScalarType() == MVT::i32;
1777
}]>;
1778
 
1779
// Zero extending masked gather fragments.
1780
def zext_masked_gather_i8 :
1781
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1782
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1783
  auto MGN = cast<MaskedGatherSDNode>(N);
1784
  return MGN->getExtensionType() == ISD::ZEXTLOAD &&
1785
         MGN->getMemoryVT().getScalarType() == MVT::i8;
1786
}]>;
1787
def zext_masked_gather_i16 :
1788
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1789
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1790
  auto MGN = cast<MaskedGatherSDNode>(N);
1791
  return MGN->getExtensionType() == ISD::ZEXTLOAD &&
1792
         MGN->getMemoryVT().getScalarType() == MVT::i16;
1793
}]>;
1794
def zext_masked_gather_i32 :
1795
  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1796
          (masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{
1797
  auto MGN = cast<MaskedGatherSDNode>(N);
1798
  return MGN->getExtensionType() == ISD::ZEXTLOAD &&
1799
         MGN->getMemoryVT().getScalarType() == MVT::i32;
1800
}]>;
1801
 
1802
// Any/Zero extending masked gather fragments.
1803
def azext_masked_gather_i8 :
1804
  PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1805
           [(ext_masked_gather_i8 node:$def, node:$pred, node:$ptr, node:$idx),
1806
            (zext_masked_gather_i8 node:$def, node:$pred, node:$ptr, node:$idx)]>;
1807
def azext_masked_gather_i16 :
1808
  PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1809
           [(ext_masked_gather_i16 node:$def, node:$pred, node:$ptr, node:$idx),
1810
            (zext_masked_gather_i16 node:$def, node:$pred, node:$ptr, node:$idx)]>;
1811
def azext_masked_gather_i32 :
1812
  PatFrags<(ops node:$def, node:$pred, node:$ptr, node:$idx),
1813
           [(ext_masked_gather_i32 node:$def, node:$pred, node:$ptr, node:$idx),
1814
            (zext_masked_gather_i32 node:$def, node:$pred, node:$ptr, node:$idx)]>;
1815
 
1816
def nontrunc_masked_scatter :
1817
  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
1818
          (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
1819
  return !cast<MaskedScatterSDNode>(N)->isTruncatingStore();
1820
}]>;
1821
 
1822
// Truncating masked scatter fragments.
1823
def trunc_masked_scatter_i8 :
1824
  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
1825
          (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
1826
  auto MSN = cast<MaskedScatterSDNode>(N);
1827
  return MSN->isTruncatingStore() &&
1828
         MSN->getMemoryVT().getScalarType() == MVT::i8;
1829
}]>;
1830
def trunc_masked_scatter_i16 :
1831
  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
1832
          (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
1833
  auto MSN = cast<MaskedScatterSDNode>(N);
1834
  return MSN->isTruncatingStore() &&
1835
         MSN->getMemoryVT().getScalarType() == MVT::i16;
1836
}]>;
1837
def trunc_masked_scatter_i32 :
1838
  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
1839
          (masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
1840
  auto MSN = cast<MaskedScatterSDNode>(N);
1841
  return MSN->isTruncatingStore() &&
1842
         MSN->getMemoryVT().getScalarType() == MVT::i32;
1843
}]>;
1844
 
1845
//===----------------------------------------------------------------------===//
1846
// Selection DAG Pattern Support.
1847
//
1848
// Patterns are what are actually matched against by the target-flavored
1849
// instruction selection DAG.  Instructions defined by the target implicitly
1850
// define patterns in most cases, but patterns can also be explicitly added when
1851
// an operation is defined by a sequence of instructions (e.g. loading a large
1852
// immediate value on RISC targets that do not support immediates as large as
1853
// their GPRs).
1854
//
1855
 
1856
class Pattern<dag patternToMatch, list<dag> resultInstrs> {
1857
  dag             PatternToMatch  = patternToMatch;
1858
  list<dag>       ResultInstrs    = resultInstrs;
1859
  list<Predicate> Predicates      = [];  // See class Instruction in Target.td.
1860
  int             AddedComplexity = 0;   // See class Instruction in Target.td.
1861
}
1862
 
1863
// Pat - A simple (but common) form of a pattern, which produces a simple result
1864
// not needing a full list.
1865
class Pat<dag pattern, dag result> : Pattern<pattern, [result]>;
1866
 
1867
//===----------------------------------------------------------------------===//
1868
// Complex pattern definitions.
1869
//
1870
 
1871
// Complex patterns, e.g. X86 addressing mode, requires pattern matching code
1872
// in C++. NumOperands is the number of operands returned by the select function;
1873
// SelectFunc is the name of the function used to pattern match the max. pattern;
1874
// RootNodes are the list of possible root nodes of the sub-dags to match.
1875
// e.g. X86 addressing mode - def addr : ComplexPattern<4, "SelectAddr", [add]>;
1876
//
1877
class ComplexPattern<ValueType ty, int numops, string fn,
1878
                     list<SDNode> roots = [], list<SDNodeProperty> props = [],
1879
                     int complexity = -1> {
1880
  ValueType Ty = ty;
1881
  int NumOperands = numops;
1882
  string SelectFunc = fn;
1883
  list<SDNode> RootNodes = roots;
1884
  list<SDNodeProperty> Properties = props;
1885
  int Complexity = complexity;
1886
}