Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file defines all of the AARCH64-specific intrinsics.

//

//===----------------------------------------------------------------------===//

let TargetPrefix = "aarch64" in {

def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],

                                 [IntrNoFree, IntrWillReturn]>;

def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],

                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],

                                 [IntrNoFree, IntrWillReturn]>;

def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],

                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],

                                 [IntrNoFree, IntrWillReturn]>;

def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],

                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],

                               [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],

                               [IntrNoFree, IntrWillReturn]>;

def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],

                                  [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],

                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_clrex : Intrinsic<[]>;

def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,

                                LLVMMatchType<0>], [IntrNoMem]>;

def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,

                                LLVMMatchType<0>], [IntrNoMem]>;

def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;

def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;

def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;

def int_aarch64_frint32z

    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],

                            [ IntrNoMem ]>;

def int_aarch64_frint64z

    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],

                            [ IntrNoMem ]>;

def int_aarch64_frint32x

    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],

                            [ IntrNoMem ]>;

def int_aarch64_frint64x

    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],

                            [ IntrNoMem ]>;

//===----------------------------------------------------------------------===//

// HINT

def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;

def int_aarch64_break : Intrinsic<[], [llvm_i32_ty],

    [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>;

def int_aarch64_prefetch : Intrinsic<[],

    [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],

    [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, ReadOnly<ArgIndex<0>>,

     ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>

     ]>,

    ClangBuiltin<"__builtin_arm_prefetch">;

//===----------------------------------------------------------------------===//

// Data Barrier Instructions

def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,

                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;

def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,

                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;

def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,

                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;

// A space-consuming intrinsic primarily for testing block and jump table

// placements. The first argument is the number of bytes this "instruction"

// takes up, the second and return value are essentially chains, used to force

// ordering during ISel.

def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;

}

//===----------------------------------------------------------------------===//

// Advanced SIMD (NEON)

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_2Scalar_Float_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_FPToIntRounding_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;

  class AdvSIMD_1IntArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_1FloatArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Expand_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Long_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;

  class AdvSIMD_1IntArg_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_any_ty], [llvm_any_ty], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Int_Across_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;

  class AdvSIMD_1VectorArg_Float_Across_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;

  class AdvSIMD_2IntArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2FloatArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Compare_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_2Arg_FloatCompare_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Long_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMTruncatedType<0>, LLVMTruncatedType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Wide_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, LLVMTruncatedType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMExtendedType<0>, LLVMExtendedType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty],

                [LLVMExtendedType<0>, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [llvm_anyvector_ty],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMTruncatedType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMTruncatedType<0>, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArg_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty],

                [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_3IntArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty],

                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_3VectorArg_Intrinsic

      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],

               [IntrNoMem]>;

  class AdvSIMD_3VectorArg_Scalar_Intrinsic

      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

               [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],

               [IntrNoMem]>;

  class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic

      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

               [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,

                LLVMMatchType<1>], [IntrNoMem]>;

  class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_CvtFxToFP_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_CvtFPToFx_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],

                [IntrNoMem]>;

  class AdvSIMD_1Arg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_Dot_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_FP16FML_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_MatMul_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_FML_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],

                [IntrNoMem]>;

  class AdvSIMD_BF16FML_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],

                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],

                [IntrNoMem]>;

}

// Arithmetic ops

let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {

  // Vector Add Across Lanes

  def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Vector Long Add Across Lanes

  def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  // Vector Halving Add

  def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Rounding Halving Add

  def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Saturating Add

  def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;

  // Vector Add High-Half

  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that

  // header is no longer supported.

  def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Rounding Add High-Half

  def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Saturating Doubling Multiply High

  def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;

  def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;

  // Vector Saturating Rounding Doubling Multiply High

  def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;

  def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;

  def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic;

  def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic;

  // Vector Polynominal Multiply

  def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Long Multiply

  def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;

  def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;

  def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;

  // 64-bit polynomial multiply really returns an i128, which is not legal. Fake

  // it with a v16i8.

  def int_aarch64_neon_pmull64 :

        DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;

  // Vector Extending Multiply

  def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {

    let IntrProperties = [IntrNoMem, Commutative];

  }

  // Vector Saturating Doubling Long Multiply

  def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;

  def int_aarch64_neon_sqdmulls_scalar

    : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;

  // Vector Halving Subtract

  def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Saturating Subtract

  def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;

  // Vector Subtract High-Half

  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that

  // header is no longer supported.

  def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Rounding Subtract High-Half

  def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Compare Absolute Greater-than-or-equal

  def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;

  // Vector Compare Absolute Greater-than

  def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;

  // Vector Absolute Difference

  def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;

  // Scalar Absolute Difference

  def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;

  // Vector Max

  def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;

  def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Max Across Lanes

  def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Vector Min

  def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;

  def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Min/Max Number

  def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;

  def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;

  // Vector Min Across Lanes

  def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Pairwise Add

  def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;

  // Long Pairwise Add

  // FIXME: In theory, we shouldn't need intrinsics for saddlp or

  // uaddlp, but tblgen's type inference currently can't handle the

  // pattern fragments this ends up generating.

  def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;

  def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;

  // Folding Maximum

  def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;

  // Folding Minimum

  def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;

  // Reciprocal Estimate/Step

  def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;

  def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;

  // Reciprocal Exponent

  def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Saturating Shift Left

  def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Rounding Shift Left

  def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Saturating Rounding Shift Left

  def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Signed->Unsigned Shift Left by Constant

  def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;

  // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant

  def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const

  def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Narrowing Shift Right by Constant

  def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Rounding Narrowing Shift Right by Constant

  def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Rounding Narrowing Saturating Shift Right by Constant

  def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Shift Left

  def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;

  def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Widening Shift Left by Constant

  def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;

  def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;

  def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;

  // Vector Shift Right by Constant and Insert

  def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;

  // Vector Shift Left by Constant and Insert

  def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;

  // Vector Saturating Narrow

  def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;

  def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;

  def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;

  def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;

  // Vector Saturating Extract and Unsigned Narrow

  def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;

  def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;

  // Vector Absolute Value

  def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;

  // Vector Saturating Absolute Value

  def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;

  // Vector Saturating Negation

  def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;

  // Vector Count Leading Sign Bits

  def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;

  // Vector Reciprocal Estimate

  def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;

  def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Square Root Estimate

  def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;

  def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Conversions Between Half-Precision and Single-Precision.

  def int_aarch64_neon_vcvtfp2hf

    : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;

  def int_aarch64_neon_vcvthf2fp

    : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;

  // Vector Conversions Between Floating-point and Fixed-point.

  def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;

  def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;

  def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;

  def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;

  // Vector FP->Int Conversions

  def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;

  def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;

  // v8.5-A Vector FP Rounding

  def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic;

  def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic;

  def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic;

  def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic;

  // Scalar FP->Int conversions

  // Vector FP Inexact Narrowing

  def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;

  // Scalar FP Inexact Narrowing

  def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty],

                                        [IntrNoMem]>;

  // v8.2-A Dot Product

  def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;

  def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;

  // v8.6-A Matrix Multiply Intrinsics

  def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;

  def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;

  def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;

  def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;

  def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;

  def int_aarch64_neon_bfmmla

    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],

                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],

                [IntrNoMem]>;

  def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic;

  def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic;

  // v8.6-A Bfloat Intrinsics

  def int_aarch64_neon_bfcvt

    : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;

  def int_aarch64_neon_bfcvtn

    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;

  def int_aarch64_neon_bfcvtn2

    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty],

                [llvm_v8bf16_ty, llvm_v4f32_ty],

                [IntrNoMem]>;

  // v8.2-A FP16 Fused Multiply-Add Long

  def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;

  def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;

  def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;

  def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;

  // v8.3-A Floating-point complex add

  def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;

  def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;

  def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;

  def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;

}

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_2Vector2Index_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],

                [IntrNoMem]>;

}

// Vector element to element moves

def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_1Vec_Load_Intrinsic

      : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],

                  [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_1Vec_Store_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],

                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;

  class AdvSIMD_2Vec_Load_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],

                [LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_2Vec_Load_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],

                [LLVMMatchType<0>, llvm_anyvector_ty,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_2Vec_Store_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,

                     LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;

  class AdvSIMD_2Vec_Store_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;

  class AdvSIMD_3Vec_Load_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],

                [LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_3Vec_Load_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],

                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_3Vec_Store_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,

                     LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;

  class AdvSIMD_3Vec_Store_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,

                 LLVMMatchType<0>, LLVMMatchType<0>,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;

  class AdvSIMD_4Vec_Load_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,

                 LLVMMatchType<0>, llvm_anyvector_ty],

                [LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_4Vec_Load_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,

                 LLVMMatchType<0>, LLVMMatchType<0>],

                [LLVMMatchType<0>, LLVMMatchType<0>,

                 LLVMMatchType<0>, llvm_anyvector_ty,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_4Vec_Store_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,

                 LLVMMatchType<0>, LLVMMatchType<0>,

                 LLVMAnyPointerType<LLVMMatchType<0>>],

                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;

  class AdvSIMD_4Vec_Store_Lane_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,

                 LLVMMatchType<0>, LLVMMatchType<0>,

                 llvm_i64_ty, llvm_anyptr_ty],

                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;

}

// Memory ops

def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;

def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;

def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;

def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;

def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;

def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;

def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;

def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;

def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;

def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;

def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;

def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;

def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;

def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;

def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;

def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;

def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;

def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_Tbl1_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbl2_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_Tbl3_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbl4_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbx1_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbx2_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbx3_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,

                 llvm_v16i8_ty, LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Tbx4_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,

                 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],

                [IntrNoMem]>;

}

def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;

def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;

def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;

def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;

def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;

def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;

def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;

def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;

let TargetPrefix = "aarch64" in {

  class FPCR_Get_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;

  class FPCR_Set_Intrinsic

    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>;

  class RNDR_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>;

}

// FPCR

def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;

def int_aarch64_set_fpcr : FPCR_Set_Intrinsic;

// Armv8.5-A Random number generation intrinsics

def int_aarch64_rndr : RNDR_Intrinsic;

def int_aarch64_rndrrs : RNDR_Intrinsic;

let TargetPrefix = "aarch64" in {

  class Crypto_AES_DataKey_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;

  class Crypto_AES_Data_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;

  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule

  // (v4i32).

  class Crypto_SHA_5Hash4Schedule_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],

                [IntrNoMem]>;

  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule

  // (v4i32).

  class Crypto_SHA_1Hash_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;

  // SHA intrinsic taking 8 words of the schedule

  class Crypto_SHA_8Schedule_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;

  // SHA intrinsic taking 12 words of the schedule

  class Crypto_SHA_12Schedule_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],

                [IntrNoMem]>;

  // SHA intrinsic taking 8 words of the hash and 4 of the schedule.

  class Crypto_SHA_8Hash4Schedule_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],

                [IntrNoMem]>;

  // SHA512 intrinsic taking 2 arguments

  class Crypto_SHA512_2Arg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;

  // SHA512 intrinsic taking 3 Arguments

  class Crypto_SHA512_3Arg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty],

                [IntrNoMem]>;

  // SHA3 Intrinsics taking 3 arguments

  class Crypto_SHA3_3Arg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],

               [IntrNoMem]>;

  // SHA3 Intrinsic taking 2 arguments

  class Crypto_SHA3_2Arg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],

               [IntrNoMem]>;

  // SHA3 Intrinsic taking 3 Arguments 1 immediate

  class Crypto_SHA3_2ArgImm_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty],

               [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class Crypto_SM3_3Vector_Intrinsic

    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],

                [IntrNoMem]>;

  class Crypto_SM3_3VectorIndexed_Intrinsic

    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty],

                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class Crypto_SM4_2Vector_Intrinsic

    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;

}

// AES

def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;

def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;

def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;

def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;

// SHA1

def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;

def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;

def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;

def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;

def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;

def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;

// SHA256

def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;

def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;

def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;

def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;

//SHA3

def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic;

def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic;

def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic;

def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic;

def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic;

def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic;

// SHA512

def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic;

def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic;

def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic;

def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic;

//SM3 & SM4

def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic;

def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic;

def int_aarch64_crypto_sm3ss1    : Crypto_SM3_3Vector_Intrinsic;

def int_aarch64_crypto_sm3tt1a   : Crypto_SM3_3VectorIndexed_Intrinsic;

def int_aarch64_crypto_sm3tt1b   : Crypto_SM3_3VectorIndexed_Intrinsic;

def int_aarch64_crypto_sm3tt2a   : Crypto_SM3_3VectorIndexed_Intrinsic;

def int_aarch64_crypto_sm3tt2b   : Crypto_SM3_3VectorIndexed_Intrinsic;

def int_aarch64_crypto_sm4e      : Crypto_SM4_2Vector_Intrinsic;

def int_aarch64_crypto_sm4ekey   : Crypto_SM4_2Vector_Intrinsic;

//===----------------------------------------------------------------------===//

// CRC32

let TargetPrefix = "aarch64" in {

def int_aarch64_crc32b  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32h  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32w  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],

    [IntrNoMem]>;

def int_aarch64_crc32x  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],

    [IntrNoMem]>;

def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],

    [IntrNoMem]>;

}

//===----------------------------------------------------------------------===//

// Memory Tagging Extensions (MTE) Intrinsics

let TargetPrefix = "aarch64" in {

def int_aarch64_irg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],

    [IntrNoMem, IntrHasSideEffects]>;

def int_aarch64_addg  : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],

    [IntrNoMem]>;

def int_aarch64_gmi   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],

    [IntrNoMem]>;

def int_aarch64_ldg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],

    [IntrReadMem]>;

def int_aarch64_stg   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],

    [IntrWriteMem]>;

def int_aarch64_subp :  DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],

    [IntrNoMem]>;

// The following are codegen-only intrinsics for stack instrumentation.

// Generate a randomly tagged stack base pointer.

def int_aarch64_irg_sp   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty],

    [IntrNoMem, IntrHasSideEffects]>;

// Transfer pointer tag with offset.

// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where

// * address is the address in ptr0

// * tag is a function of (tag in baseptr, tag_offset).

// ** Beware, this is not the same function as implemented by the ADDG instruction!

//    Backend optimizations may change tag_offset; the only guarantee is that calls

//    to tagp with the same pair of (baseptr, tag_offset) will produce pointers

//    with the same tag value, assuming the set of excluded tags has not changed.

// Address bits in baseptr and tag bits in ptr0 are ignored.

// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as

//   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset

// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.

def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],

    [IntrNoMem, ImmArg<ArgIndex<2>>]>;

// Update allocation tags for the memory range to match the tag in the pointer argument.

def int_aarch64_settag  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],

    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

// Update allocation tags for the memory range to match the tag in the pointer argument,

// and set memory contents to zero.

def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],

    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.

def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],

    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

}

//===----------------------------------------------------------------------===//

// Memory Operations (MOPS) Intrinsics

let TargetPrefix = "aarch64" in {

  // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64

  def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty],

      [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

}

// Transactional Memory Extension (TME) Intrinsics

let TargetPrefix = "aarch64" in {

def int_aarch64_tstart  : ClangBuiltin<"__builtin_arm_tstart">,

                         Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;

def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;

def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">,

                          Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>;

def int_aarch64_ttest   : ClangBuiltin<"__builtin_arm_ttest">,

                          Intrinsic<[llvm_i64_ty], [],

                                    [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;

// Armv8.7-A load/store 64-byte intrinsics

defvar data512 = !listsplat(llvm_i64_ty, 8);

def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;

def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;

def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;

def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;

}

def llvm_nxv1i1_ty  : LLVMType<nxv1i1>;

def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;

def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;

def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;

def llvm_nxv16i1_ty : LLVMType<nxv16i1>;

def llvm_nxv16i8_ty : LLVMType<nxv16i8>;

def llvm_nxv4i32_ty : LLVMType<nxv4i32>;

def llvm_nxv2i64_ty : LLVMType<nxv2i64>;

def llvm_nxv8f16_ty : LLVMType<nxv8f16>;

def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;

def llvm_nxv4f32_ty : LLVMType<nxv4f32>;

def llvm_nxv2f64_ty : LLVMType<nxv2f64>;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_1Vec_PredLoad_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_2Vec_PredLoad_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_3Vec_PredLoad_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_4Vec_PredLoad_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,

                 LLVMMatchType<0>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrInaccessibleMemOrArgMemOnly]>;

  class AdvSIMD_1Vec_PredStore_Intrinsic

    : DefaultAttrsIntrinsic<[],

                [llvm_anyvector_ty,

                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMPointerToElt<0>],

                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;

  class AdvSIMD_2Vec_PredStore_Intrinsic

      : DefaultAttrsIntrinsic<[],

                  [llvm_anyvector_ty, LLVMMatchType<0>,

                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],

                  [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;

  class AdvSIMD_3Vec_PredStore_Intrinsic

      : DefaultAttrsIntrinsic<[],

                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,

                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],

                  [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;

  class AdvSIMD_4Vec_PredStore_Intrinsic

      : DefaultAttrsIntrinsic<[],

                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,

                   LLVMMatchType<0>,

                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],

                  [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;

  class AdvSIMD_SVE_Index_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMVectorElementType<0>,

                 LLVMVectorElementType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Merged1VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>,

                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_2VectorArgIndexed_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>,

                 LLVMMatchType<0>,

                 llvm_i32_ty],

                [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_3VectorArgIndexed_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>,

                 LLVMMatchType<0>,

                 LLVMMatchType<0>,

                 llvm_i32_ty],

                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class AdvSIMD_Pred1VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Pred2VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_Pred3VectorArg_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>,

                 LLVMMatchType<0>,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_SVE_Compare_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 llvm_anyvector_ty,

                 LLVMMatchType<0>],

                [IntrNoMem]>;

  class AdvSIMD_SVE_CompareWide_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 llvm_anyvector_ty,

                 llvm_nxv2i64_ty],

                [IntrNoMem]>;

  class AdvSIMD_SVE_Saturating_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>,

                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],

                [IntrNoMem]>;

  class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMMatchType<0>,

                 llvm_i32_ty,

                 llvm_i32_ty],

                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>

    : DefaultAttrsIntrinsic<[T],

                [T, llvm_anyvector_ty],

                [IntrNoMem]>;

  class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>

    : DefaultAttrsIntrinsic<[T],

                [T, llvm_i32_ty, llvm_i32_ty],

                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_CNT_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>],

                [LLVMVectorOfBitcastsToInt<0>,

                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 llvm_anyvector_ty],

                [IntrNoMem]>;

  class AdvSIMD_SVE_ReduceWithInit_Intrinsic

    : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMVectorElementType<0>,

                 llvm_anyvector_ty],

                [IntrNoMem]>;

  class AdvSIMD_SVE_ShiftByImm_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>,

                 llvm_i32_ty],

                [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_ShiftWide_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,

                 LLVMMatchType<0>,

                 llvm_nxv2i64_ty],

                [IntrNoMem]>;

  class AdvSIMD_SVE_Unpack_Intrinsic

    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],

               [LLVMSubdivide2VectorType<0>],