Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

Rev

Blame | Last modification | View Log | Download | RSS feed

  1. /*===---- xmmintrin.h - Implementation of SSE intrinsics on PowerPC --------===
  2.  *
  3.  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4.  * See https://llvm.org/LICENSE.txt for license information.
  5.  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6.  *
  7.  *===-----------------------------------------------------------------------===
  8.  */
  9.  
  10. /* Implemented from the specification included in the Intel C++ Compiler
  11.    User Guide and Reference, version 9.0.  */
  12.  
  13. #ifndef NO_WARN_X86_INTRINSICS
  14. /* This header file is to help porting code using Intel intrinsics
  15.    explicitly from x86_64 to powerpc64/powerpc64le.
  16.  
  17.    Since X86 SSE intrinsics mainly handles __m128 type, PowerPC
  18.    VMX/VSX ISA is a good match for vector float SIMD operations.
  19.    However scalar float operations in vector (XMM) registers require
  20.    the POWER8 VSX ISA (2.07) level. There are differences for data
  21.    format and placement of float scalars in the vector register, which
  22.    require extra steps to match SSE scalar float semantics on POWER.
  23.  
  24.    It should be noted that there's much difference between X86_64's
  25.    MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use
  26.    portable <fenv.h> instead of access MXSCR directly.
  27.  
  28.    Most SSE scalar float intrinsic operations can be performed more
  29.    efficiently as C language float scalar operations or optimized to
  30.    use vector SIMD operations. We recommend this for new applications. */
  31. #error                                                                         \
  32.     "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."
  33. #endif
  34.  
  35. #ifndef XMMINTRIN_H_
  36. #define XMMINTRIN_H_
  37.  
  38. #if defined(__powerpc64__) &&                                                  \
  39.     (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX))
  40.  
  41. /* Define four value permute mask */
  42. #define _MM_SHUFFLE(w, x, y, z) (((w) << 6) | ((x) << 4) | ((y) << 2) | (z))
  43.  
  44. #include <altivec.h>
  45.  
  46. /* Avoid collisions between altivec.h and strict adherence to C++ and
  47.    C11 standards.  This should eventually be done inside altivec.h itself,
  48.    but only after testing a full distro build.  */
  49. #if defined(__STRICT_ANSI__) &&                                                \
  50.     (defined(__cplusplus) ||                                                   \
  51.      (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L))
  52. #undef vector
  53. #undef pixel
  54. #undef bool
  55. #endif
  56.  
  57. /* We need type definitions from the MMX header file.  */
  58. #include <mmintrin.h>
  59.  
  60. /* Get _mm_malloc () and _mm_free ().  */
  61. #if __STDC_HOSTED__
  62. #include <mm_malloc.h>
  63. #endif
  64.  
  65. /* The Intel API is flexible enough that we must allow aliasing with other
  66.    vector types, and their scalar components.  */
  67. typedef vector float __m128 __attribute__((__may_alias__));
  68.  
  69. /* Unaligned version of the same type.  */
  70. typedef vector float __m128_u __attribute__((__may_alias__, __aligned__(1)));
  71.  
  72. /* Internal data types for implementing the intrinsics.  */
  73. typedef vector float __v4sf;
  74.  
  75. /* Create an undefined vector.  */
  76. extern __inline __m128
  77.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  78.     _mm_undefined_ps(void) {
  79.   __m128 __Y = __Y;
  80.   return __Y;
  81. }
  82.  
  83. /* Create a vector of zeros.  */
  84. extern __inline __m128
  85.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  86.     _mm_setzero_ps(void) {
  87.   return __extension__(__m128){0.0f, 0.0f, 0.0f, 0.0f};
  88. }
  89.  
  90. /* Load four SPFP values from P.  The address must be 16-byte aligned.  */
  91. extern __inline __m128
  92.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  93.     _mm_load_ps(float const *__P) {
  94.   return ((__m128)vec_ld(0, (__v4sf *)__P));
  95. }
  96.  
  97. /* Load four SPFP values from P.  The address need not be 16-byte aligned.  */
  98. extern __inline __m128
  99.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  100.     _mm_loadu_ps(float const *__P) {
  101.   return (vec_vsx_ld(0, __P));
  102. }
  103.  
  104. /* Load four SPFP values in reverse order.  The address must be aligned.  */
  105. extern __inline __m128
  106.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  107.     _mm_loadr_ps(float const *__P) {
  108.   __v4sf __tmp;
  109.   __m128 __result;
  110.   static const __vector unsigned char __permute_vector = {
  111.       0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B,
  112.       0x14, 0x15, 0x16, 0x17, 0x10, 0x11, 0x12, 0x13};
  113.  
  114.   __tmp = vec_ld(0, (__v4sf *)__P);
  115.   __result = (__m128)vec_perm(__tmp, __tmp, __permute_vector);
  116.   return __result;
  117. }
  118.  
  119. /* Create a vector with all four elements equal to F.  */
  120. extern __inline __m128
  121.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  122.     _mm_set1_ps(float __F) {
  123.   return __extension__(__m128)(__v4sf){__F, __F, __F, __F};
  124. }
  125.  
  126. extern __inline __m128
  127.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  128.     _mm_set_ps1(float __F) {
  129.   return _mm_set1_ps(__F);
  130. }
  131.  
  132. /* Create the vector [Z Y X W].  */
  133. extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__,
  134.                                       __artificial__))
  135. _mm_set_ps(const float __Z, const float __Y, const float __X, const float __W) {
  136.   return __extension__(__m128)(__v4sf){__W, __X, __Y, __Z};
  137. }
  138.  
  139. /* Create the vector [W X Y Z].  */
  140. extern __inline __m128
  141.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  142.     _mm_setr_ps(float __Z, float __Y, float __X, float __W) {
  143.   return __extension__(__m128)(__v4sf){__Z, __Y, __X, __W};
  144. }
  145.  
  146. /* Store four SPFP values.  The address must be 16-byte aligned.  */
  147. extern __inline void
  148.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  149.     _mm_store_ps(float *__P, __m128 __A) {
  150.   vec_st((__v4sf)__A, 0, (__v4sf *)__P);
  151. }
  152.  
  153. /* Store four SPFP values.  The address need not be 16-byte aligned.  */
  154. extern __inline void
  155.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  156.     _mm_storeu_ps(float *__P, __m128 __A) {
  157.   *(__m128_u *)__P = __A;
  158. }
  159.  
  160. /* Store four SPFP values in reverse order.  The address must be aligned.  */
  161. extern __inline void
  162.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  163.     _mm_storer_ps(float *__P, __m128 __A) {
  164.   __v4sf __tmp;
  165.   static const __vector unsigned char __permute_vector = {
  166.       0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B,
  167.       0x14, 0x15, 0x16, 0x17, 0x10, 0x11, 0x12, 0x13};
  168.  
  169.   __tmp = (__m128)vec_perm(__A, __A, __permute_vector);
  170.  
  171.   _mm_store_ps(__P, __tmp);
  172. }
  173.  
  174. /* Store the lower SPFP value across four words.  */
  175. extern __inline void
  176.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  177.     _mm_store1_ps(float *__P, __m128 __A) {
  178.   __v4sf __va = vec_splat((__v4sf)__A, 0);
  179.   _mm_store_ps(__P, __va);
  180. }
  181.  
  182. extern __inline void
  183.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  184.     _mm_store_ps1(float *__P, __m128 __A) {
  185.   _mm_store1_ps(__P, __A);
  186. }
  187.  
  188. /* Create a vector with element 0 as F and the rest zero.  */
  189. extern __inline __m128
  190.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  191.     _mm_set_ss(float __F) {
  192.   return __extension__(__m128)(__v4sf){__F, 0.0f, 0.0f, 0.0f};
  193. }
  194.  
  195. /* Sets the low SPFP value of A from the low value of B.  */
  196. extern __inline __m128
  197.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  198.     _mm_move_ss(__m128 __A, __m128 __B) {
  199.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  200.  
  201.   return (vec_sel((__v4sf)__A, (__v4sf)__B, __mask));
  202. }
  203.  
  204. /* Create a vector with element 0 as *P and the rest zero.  */
  205. extern __inline __m128
  206.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  207.     _mm_load_ss(float const *__P) {
  208.   return _mm_set_ss(*__P);
  209. }
  210.  
  211. /* Stores the lower SPFP value.  */
  212. extern __inline void
  213.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  214.     _mm_store_ss(float *__P, __m128 __A) {
  215.   *__P = ((__v4sf)__A)[0];
  216. }
  217.  
  218. /* Perform the respective operation on the lower SPFP (single-precision
  219.    floating-point) values of A and B; the upper three SPFP values are
  220.    passed through from A.  */
  221.  
  222. extern __inline __m128
  223.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  224.     _mm_add_ss(__m128 __A, __m128 __B) {
  225. #ifdef _ARCH_PWR7
  226.   __m128 __a, __b, __c;
  227.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  228.   /* PowerISA VSX does not allow partial (for just lower double)
  229.      results. So to insure we don't generate spurious exceptions
  230.      (from the upper double values) we splat the lower double
  231.      before we to the operation.  */
  232.   __a = vec_splat(__A, 0);
  233.   __b = vec_splat(__B, 0);
  234.   __c = __a + __b;
  235.   /* Then we merge the lower float result with the original upper
  236.      float elements from __A.  */
  237.   return (vec_sel(__A, __c, __mask));
  238. #else
  239.   __A[0] = __A[0] + __B[0];
  240.   return (__A);
  241. #endif
  242. }
  243.  
  244. extern __inline __m128
  245.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  246.     _mm_sub_ss(__m128 __A, __m128 __B) {
  247. #ifdef _ARCH_PWR7
  248.   __m128 __a, __b, __c;
  249.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  250.   /* PowerISA VSX does not allow partial (for just lower double)
  251.      results. So to insure we don't generate spurious exceptions
  252.      (from the upper double values) we splat the lower double
  253.      before we to the operation.  */
  254.   __a = vec_splat(__A, 0);
  255.   __b = vec_splat(__B, 0);
  256.   __c = __a - __b;
  257.   /* Then we merge the lower float result with the original upper
  258.      float elements from __A.  */
  259.   return (vec_sel(__A, __c, __mask));
  260. #else
  261.   __A[0] = __A[0] - __B[0];
  262.   return (__A);
  263. #endif
  264. }
  265.  
  266. extern __inline __m128
  267.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  268.     _mm_mul_ss(__m128 __A, __m128 __B) {
  269. #ifdef _ARCH_PWR7
  270.   __m128 __a, __b, __c;
  271.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  272.   /* PowerISA VSX does not allow partial (for just lower double)
  273.      results. So to insure we don't generate spurious exceptions
  274.      (from the upper double values) we splat the lower double
  275.      before we to the operation.  */
  276.   __a = vec_splat(__A, 0);
  277.   __b = vec_splat(__B, 0);
  278.   __c = __a * __b;
  279.   /* Then we merge the lower float result with the original upper
  280.      float elements from __A.  */
  281.   return (vec_sel(__A, __c, __mask));
  282. #else
  283.   __A[0] = __A[0] * __B[0];
  284.   return (__A);
  285. #endif
  286. }
  287.  
  288. extern __inline __m128
  289.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  290.     _mm_div_ss(__m128 __A, __m128 __B) {
  291. #ifdef _ARCH_PWR7
  292.   __m128 __a, __b, __c;
  293.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  294.   /* PowerISA VSX does not allow partial (for just lower double)
  295.      results. So to insure we don't generate spurious exceptions
  296.      (from the upper double values) we splat the lower double
  297.      before we to the operation.  */
  298.   __a = vec_splat(__A, 0);
  299.   __b = vec_splat(__B, 0);
  300.   __c = __a / __b;
  301.   /* Then we merge the lower float result with the original upper
  302.      float elements from __A.  */
  303.   return (vec_sel(__A, __c, __mask));
  304. #else
  305.   __A[0] = __A[0] / __B[0];
  306.   return (__A);
  307. #endif
  308. }
  309.  
  310. extern __inline __m128
  311.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  312.     _mm_sqrt_ss(__m128 __A) {
  313.   __m128 __a, __c;
  314.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  315.   /* PowerISA VSX does not allow partial (for just lower double)
  316.    * results. So to insure we don't generate spurious exceptions
  317.    * (from the upper double values) we splat the lower double
  318.    * before we to the operation. */
  319.   __a = vec_splat(__A, 0);
  320.   __c = vec_sqrt(__a);
  321.   /* Then we merge the lower float result with the original upper
  322.    * float elements from __A.  */
  323.   return (vec_sel(__A, __c, __mask));
  324. }
  325.  
  326. /* Perform the respective operation on the four SPFP values in A and B.  */
  327. extern __inline __m128
  328.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  329.     _mm_add_ps(__m128 __A, __m128 __B) {
  330.   return (__m128)((__v4sf)__A + (__v4sf)__B);
  331. }
  332.  
  333. extern __inline __m128
  334.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  335.     _mm_sub_ps(__m128 __A, __m128 __B) {
  336.   return (__m128)((__v4sf)__A - (__v4sf)__B);
  337. }
  338.  
  339. extern __inline __m128
  340.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  341.     _mm_mul_ps(__m128 __A, __m128 __B) {
  342.   return (__m128)((__v4sf)__A * (__v4sf)__B);
  343. }
  344.  
  345. extern __inline __m128
  346.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  347.     _mm_div_ps(__m128 __A, __m128 __B) {
  348.   return (__m128)((__v4sf)__A / (__v4sf)__B);
  349. }
  350.  
  351. extern __inline __m128
  352.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  353.     _mm_sqrt_ps(__m128 __A) {
  354.   return (vec_sqrt((__v4sf)__A));
  355. }
  356.  
  357. extern __inline __m128
  358.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  359.     _mm_rcp_ps(__m128 __A) {
  360.   return (vec_re((__v4sf)__A));
  361. }
  362.  
  363. extern __inline __m128
  364.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  365.     _mm_rsqrt_ps(__m128 __A) {
  366.   return (vec_rsqrte(__A));
  367. }
  368.  
  369. extern __inline __m128
  370.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  371.     _mm_rcp_ss(__m128 __A) {
  372.   __m128 __a, __c;
  373.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  374.   /* PowerISA VSX does not allow partial (for just lower double)
  375.    * results. So to insure we don't generate spurious exceptions
  376.    * (from the upper double values) we splat the lower double
  377.    * before we to the operation. */
  378.   __a = vec_splat(__A, 0);
  379.   __c = _mm_rcp_ps(__a);
  380.   /* Then we merge the lower float result with the original upper
  381.    * float elements from __A.  */
  382.   return (vec_sel(__A, __c, __mask));
  383. }
  384.  
  385. extern __inline __m128
  386.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  387.     _mm_rsqrt_ss(__m128 __A) {
  388.   __m128 __a, __c;
  389.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  390.   /* PowerISA VSX does not allow partial (for just lower double)
  391.    * results. So to insure we don't generate spurious exceptions
  392.    * (from the upper double values) we splat the lower double
  393.    * before we to the operation. */
  394.   __a = vec_splat(__A, 0);
  395.   __c = vec_rsqrte(__a);
  396.   /* Then we merge the lower float result with the original upper
  397.    * float elements from __A.  */
  398.   return (vec_sel(__A, __c, __mask));
  399. }
  400.  
  401. extern __inline __m128
  402.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  403.     _mm_min_ss(__m128 __A, __m128 __B) {
  404.   __v4sf __a, __b, __c;
  405.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  406.   /* PowerISA VSX does not allow partial (for just lower float)
  407.    * results. So to insure we don't generate spurious exceptions
  408.    * (from the upper float values) we splat the lower float
  409.    * before we to the operation. */
  410.   __a = vec_splat((__v4sf)__A, 0);
  411.   __b = vec_splat((__v4sf)__B, 0);
  412.   __c = vec_min(__a, __b);
  413.   /* Then we merge the lower float result with the original upper
  414.    * float elements from __A.  */
  415.   return (vec_sel((__v4sf)__A, __c, __mask));
  416. }
  417.  
  418. extern __inline __m128
  419.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  420.     _mm_max_ss(__m128 __A, __m128 __B) {
  421.   __v4sf __a, __b, __c;
  422.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  423.   /* PowerISA VSX does not allow partial (for just lower float)
  424.    * results. So to insure we don't generate spurious exceptions
  425.    * (from the upper float values) we splat the lower float
  426.    * before we to the operation. */
  427.   __a = vec_splat(__A, 0);
  428.   __b = vec_splat(__B, 0);
  429.   __c = vec_max(__a, __b);
  430.   /* Then we merge the lower float result with the original upper
  431.    * float elements from __A.  */
  432.   return (vec_sel((__v4sf)__A, __c, __mask));
  433. }
  434.  
  435. extern __inline __m128
  436.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  437.     _mm_min_ps(__m128 __A, __m128 __B) {
  438.   __vector __bool int __m = vec_cmpgt((__v4sf)__B, (__v4sf)__A);
  439.   return vec_sel(__B, __A, __m);
  440. }
  441.  
  442. extern __inline __m128
  443.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  444.     _mm_max_ps(__m128 __A, __m128 __B) {
  445.   __vector __bool int __m = vec_cmpgt((__v4sf)__A, (__v4sf)__B);
  446.   return vec_sel(__B, __A, __m);
  447. }
  448.  
  449. /* Perform logical bit-wise operations on 128-bit values.  */
  450. extern __inline __m128
  451.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  452.     _mm_and_ps(__m128 __A, __m128 __B) {
  453.   return ((__m128)vec_and((__v4sf)__A, (__v4sf)__B));
  454.   //  return __builtin_ia32_andps (__A, __B);
  455. }
  456.  
  457. extern __inline __m128
  458.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  459.     _mm_andnot_ps(__m128 __A, __m128 __B) {
  460.   return ((__m128)vec_andc((__v4sf)__B, (__v4sf)__A));
  461. }
  462.  
  463. extern __inline __m128
  464.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  465.     _mm_or_ps(__m128 __A, __m128 __B) {
  466.   return ((__m128)vec_or((__v4sf)__A, (__v4sf)__B));
  467. }
  468.  
  469. extern __inline __m128
  470.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  471.     _mm_xor_ps(__m128 __A, __m128 __B) {
  472.   return ((__m128)vec_xor((__v4sf)__A, (__v4sf)__B));
  473. }
  474.  
  475. /* Perform a comparison on the four SPFP values of A and B.  For each
  476.    element, if the comparison is true, place a mask of all ones in the
  477.    result, otherwise a mask of zeros.  */
  478. extern __inline __m128
  479.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  480.     _mm_cmpeq_ps(__m128 __A, __m128 __B) {
  481.   return ((__m128)vec_cmpeq((__v4sf)__A, (__v4sf)__B));
  482. }
  483.  
  484. extern __inline __m128
  485.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  486.     _mm_cmplt_ps(__m128 __A, __m128 __B) {
  487.   return ((__m128)vec_cmplt((__v4sf)__A, (__v4sf)__B));
  488. }
  489.  
  490. extern __inline __m128
  491.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  492.     _mm_cmple_ps(__m128 __A, __m128 __B) {
  493.   return ((__m128)vec_cmple((__v4sf)__A, (__v4sf)__B));
  494. }
  495.  
  496. extern __inline __m128
  497.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  498.     _mm_cmpgt_ps(__m128 __A, __m128 __B) {
  499.   return ((__m128)vec_cmpgt((__v4sf)__A, (__v4sf)__B));
  500. }
  501.  
  502. extern __inline __m128
  503.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  504.     _mm_cmpge_ps(__m128 __A, __m128 __B) {
  505.   return ((__m128)vec_cmpge((__v4sf)__A, (__v4sf)__B));
  506. }
  507.  
  508. extern __inline __m128
  509.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  510.     _mm_cmpneq_ps(__m128 __A, __m128 __B) {
  511.   __v4sf __temp = (__v4sf)vec_cmpeq((__v4sf)__A, (__v4sf)__B);
  512.   return ((__m128)vec_nor(__temp, __temp));
  513. }
  514.  
  515. extern __inline __m128
  516.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  517.     _mm_cmpnlt_ps(__m128 __A, __m128 __B) {
  518.   return ((__m128)vec_cmpge((__v4sf)__A, (__v4sf)__B));
  519. }
  520.  
  521. extern __inline __m128
  522.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  523.     _mm_cmpnle_ps(__m128 __A, __m128 __B) {
  524.   return ((__m128)vec_cmpgt((__v4sf)__A, (__v4sf)__B));
  525. }
  526.  
  527. extern __inline __m128
  528.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  529.     _mm_cmpngt_ps(__m128 __A, __m128 __B) {
  530.   return ((__m128)vec_cmple((__v4sf)__A, (__v4sf)__B));
  531. }
  532.  
  533. extern __inline __m128
  534.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  535.     _mm_cmpnge_ps(__m128 __A, __m128 __B) {
  536.   return ((__m128)vec_cmplt((__v4sf)__A, (__v4sf)__B));
  537. }
  538.  
  539. extern __inline __m128
  540.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  541.     _mm_cmpord_ps(__m128 __A, __m128 __B) {
  542.   __vector unsigned int __a, __b;
  543.   __vector unsigned int __c, __d;
  544.   static const __vector unsigned int __float_exp_mask = {
  545.       0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  546.  
  547.   __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  548.   __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  549.   __c = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __a);
  550.   __d = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __b);
  551.   return ((__m128)vec_and(__c, __d));
  552. }
  553.  
  554. extern __inline __m128
  555.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  556.     _mm_cmpunord_ps(__m128 __A, __m128 __B) {
  557.   __vector unsigned int __a, __b;
  558.   __vector unsigned int __c, __d;
  559.   static const __vector unsigned int __float_exp_mask = {
  560.       0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  561.  
  562.   __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  563.   __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  564.   __c = (__vector unsigned int)vec_cmpgt(__a, __float_exp_mask);
  565.   __d = (__vector unsigned int)vec_cmpgt(__b, __float_exp_mask);
  566.   return ((__m128)vec_or(__c, __d));
  567. }
  568.  
  569. /* Perform a comparison on the lower SPFP values of A and B.  If the
  570.    comparison is true, place a mask of all ones in the result, otherwise a
  571.    mask of zeros.  The upper three SPFP values are passed through from A.  */
  572. extern __inline __m128
  573.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  574.     _mm_cmpeq_ss(__m128 __A, __m128 __B) {
  575.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  576.   __v4sf __a, __b, __c;
  577.   /* PowerISA VMX does not allow partial (for just element 0)
  578.    * results. So to insure we don't generate spurious exceptions
  579.    * (from the upper elements) we splat the lower float
  580.    * before we to the operation. */
  581.   __a = vec_splat((__v4sf)__A, 0);
  582.   __b = vec_splat((__v4sf)__B, 0);
  583.   __c = (__v4sf)vec_cmpeq(__a, __b);
  584.   /* Then we merge the lower float result with the original upper
  585.    * float elements from __A.  */
  586.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  587. }
  588.  
  589. extern __inline __m128
  590.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  591.     _mm_cmplt_ss(__m128 __A, __m128 __B) {
  592.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  593.   __v4sf __a, __b, __c;
  594.   /* PowerISA VMX does not allow partial (for just element 0)
  595.    * results. So to insure we don't generate spurious exceptions
  596.    * (from the upper elements) we splat the lower float
  597.    * before we to the operation. */
  598.   __a = vec_splat((__v4sf)__A, 0);
  599.   __b = vec_splat((__v4sf)__B, 0);
  600.   __c = (__v4sf)vec_cmplt(__a, __b);
  601.   /* Then we merge the lower float result with the original upper
  602.    * float elements from __A.  */
  603.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  604. }
  605.  
  606. extern __inline __m128
  607.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  608.     _mm_cmple_ss(__m128 __A, __m128 __B) {
  609.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  610.   __v4sf __a, __b, __c;
  611.   /* PowerISA VMX does not allow partial (for just element 0)
  612.    * results. So to insure we don't generate spurious exceptions
  613.    * (from the upper elements) we splat the lower float
  614.    * before we to the operation. */
  615.   __a = vec_splat((__v4sf)__A, 0);
  616.   __b = vec_splat((__v4sf)__B, 0);
  617.   __c = (__v4sf)vec_cmple(__a, __b);
  618.   /* Then we merge the lower float result with the original upper
  619.    * float elements from __A.  */
  620.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  621. }
  622.  
  623. extern __inline __m128
  624.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  625.     _mm_cmpgt_ss(__m128 __A, __m128 __B) {
  626.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  627.   __v4sf __a, __b, __c;
  628.   /* PowerISA VMX does not allow partial (for just element 0)
  629.    * results. So to insure we don't generate spurious exceptions
  630.    * (from the upper elements) we splat the lower float
  631.    * before we to the operation. */
  632.   __a = vec_splat((__v4sf)__A, 0);
  633.   __b = vec_splat((__v4sf)__B, 0);
  634.   __c = (__v4sf)vec_cmpgt(__a, __b);
  635.   /* Then we merge the lower float result with the original upper
  636.    * float elements from __A.  */
  637.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  638. }
  639.  
  640. extern __inline __m128
  641.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  642.     _mm_cmpge_ss(__m128 __A, __m128 __B) {
  643.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  644.   __v4sf __a, __b, __c;
  645.   /* PowerISA VMX does not allow partial (for just element 0)
  646.    * results. So to insure we don't generate spurious exceptions
  647.    * (from the upper elements) we splat the lower float
  648.    * before we to the operation. */
  649.   __a = vec_splat((__v4sf)__A, 0);
  650.   __b = vec_splat((__v4sf)__B, 0);
  651.   __c = (__v4sf)vec_cmpge(__a, __b);
  652.   /* Then we merge the lower float result with the original upper
  653.    * float elements from __A.  */
  654.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  655. }
  656.  
  657. extern __inline __m128
  658.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  659.     _mm_cmpneq_ss(__m128 __A, __m128 __B) {
  660.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  661.   __v4sf __a, __b, __c;
  662.   /* PowerISA VMX does not allow partial (for just element 0)
  663.    * results. So to insure we don't generate spurious exceptions
  664.    * (from the upper elements) we splat the lower float
  665.    * before we to the operation. */
  666.   __a = vec_splat((__v4sf)__A, 0);
  667.   __b = vec_splat((__v4sf)__B, 0);
  668.   __c = (__v4sf)vec_cmpeq(__a, __b);
  669.   __c = vec_nor(__c, __c);
  670.   /* Then we merge the lower float result with the original upper
  671.    * float elements from __A.  */
  672.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  673. }
  674.  
  675. extern __inline __m128
  676.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  677.     _mm_cmpnlt_ss(__m128 __A, __m128 __B) {
  678.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  679.   __v4sf __a, __b, __c;
  680.   /* PowerISA VMX does not allow partial (for just element 0)
  681.    * results. So to insure we don't generate spurious exceptions
  682.    * (from the upper elements) we splat the lower float
  683.    * before we to the operation. */
  684.   __a = vec_splat((__v4sf)__A, 0);
  685.   __b = vec_splat((__v4sf)__B, 0);
  686.   __c = (__v4sf)vec_cmpge(__a, __b);
  687.   /* Then we merge the lower float result with the original upper
  688.    * float elements from __A.  */
  689.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  690. }
  691.  
  692. extern __inline __m128
  693.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  694.     _mm_cmpnle_ss(__m128 __A, __m128 __B) {
  695.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  696.   __v4sf __a, __b, __c;
  697.   /* PowerISA VMX does not allow partial (for just element 0)
  698.    * results. So to insure we don't generate spurious exceptions
  699.    * (from the upper elements) we splat the lower float
  700.    * before we to the operation. */
  701.   __a = vec_splat((__v4sf)__A, 0);
  702.   __b = vec_splat((__v4sf)__B, 0);
  703.   __c = (__v4sf)vec_cmpgt(__a, __b);
  704.   /* Then we merge the lower float result with the original upper
  705.    * float elements from __A.  */
  706.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  707. }
  708.  
  709. extern __inline __m128
  710.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  711.     _mm_cmpngt_ss(__m128 __A, __m128 __B) {
  712.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  713.   __v4sf __a, __b, __c;
  714.   /* PowerISA VMX does not allow partial (for just element 0)
  715.    * results. So to insure we don't generate spurious exceptions
  716.    * (from the upper elements) we splat the lower float
  717.    * before we to the operation. */
  718.   __a = vec_splat((__v4sf)__A, 0);
  719.   __b = vec_splat((__v4sf)__B, 0);
  720.   __c = (__v4sf)vec_cmple(__a, __b);
  721.   /* Then we merge the lower float result with the original upper
  722.    * float elements from __A.  */
  723.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  724. }
  725.  
  726. extern __inline __m128
  727.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  728.     _mm_cmpnge_ss(__m128 __A, __m128 __B) {
  729.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  730.   __v4sf __a, __b, __c;
  731.   /* PowerISA VMX does not allow partial (for just element 0)
  732.    * results. So to insure we don't generate spurious exceptions
  733.    * (from the upper elements) we splat the lower float
  734.    * before we do the operation. */
  735.   __a = vec_splat((__v4sf)__A, 0);
  736.   __b = vec_splat((__v4sf)__B, 0);
  737.   __c = (__v4sf)vec_cmplt(__a, __b);
  738.   /* Then we merge the lower float result with the original upper
  739.    * float elements from __A.  */
  740.   return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
  741. }
  742.  
  743. extern __inline __m128
  744.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  745.     _mm_cmpord_ss(__m128 __A, __m128 __B) {
  746.   __vector unsigned int __a, __b;
  747.   __vector unsigned int __c, __d;
  748.   static const __vector unsigned int __float_exp_mask = {
  749.       0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  750.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  751.  
  752.   __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  753.   __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  754.   __c = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __a);
  755.   __d = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __b);
  756.   __c = vec_and(__c, __d);
  757.   /* Then we merge the lower float result with the original upper
  758.    * float elements from __A.  */
  759.   return ((__m128)vec_sel((__v4sf)__A, (__v4sf)__c, __mask));
  760. }
  761.  
  762. extern __inline __m128
  763.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  764.     _mm_cmpunord_ss(__m128 __A, __m128 __B) {
  765.   __vector unsigned int __a, __b;
  766.   __vector unsigned int __c, __d;
  767.   static const __vector unsigned int __float_exp_mask = {
  768.       0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  769.   static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  770.  
  771.   __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  772.   __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  773.   __c = (__vector unsigned int)vec_cmpgt(__a, __float_exp_mask);
  774.   __d = (__vector unsigned int)vec_cmpgt(__b, __float_exp_mask);
  775.   __c = vec_or(__c, __d);
  776.   /* Then we merge the lower float result with the original upper
  777.    * float elements from __A.  */
  778.   return ((__m128)vec_sel((__v4sf)__A, (__v4sf)__c, __mask));
  779. }
  780.  
  781. /* Compare the lower SPFP values of A and B and return 1 if true
  782.    and 0 if false.  */
  783. extern __inline int
  784.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  785.     _mm_comieq_ss(__m128 __A, __m128 __B) {
  786.   return (__A[0] == __B[0]);
  787. }
  788.  
  789. extern __inline int
  790.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  791.     _mm_comilt_ss(__m128 __A, __m128 __B) {
  792.   return (__A[0] < __B[0]);
  793. }
  794.  
  795. extern __inline int
  796.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  797.     _mm_comile_ss(__m128 __A, __m128 __B) {
  798.   return (__A[0] <= __B[0]);
  799. }
  800.  
  801. extern __inline int
  802.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  803.     _mm_comigt_ss(__m128 __A, __m128 __B) {
  804.   return (__A[0] > __B[0]);
  805. }
  806.  
  807. extern __inline int
  808.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  809.     _mm_comige_ss(__m128 __A, __m128 __B) {
  810.   return (__A[0] >= __B[0]);
  811. }
  812.  
  813. extern __inline int
  814.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  815.     _mm_comineq_ss(__m128 __A, __m128 __B) {
  816.   return (__A[0] != __B[0]);
  817. }
  818.  
  819. /* FIXME
  820.  * The __mm_ucomi??_ss implementations below are exactly the same as
  821.  * __mm_comi??_ss because GCC for PowerPC only generates unordered
  822.  * compares (scalar and vector).
  823.  * Technically __mm_comieq_ss et al should be using the ordered
  824.  * compare and signal for QNaNs.
  825.  * The __mm_ucomieq_sd et all should be OK, as is.
  826.  */
  827. extern __inline int
  828.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  829.     _mm_ucomieq_ss(__m128 __A, __m128 __B) {
  830.   return (__A[0] == __B[0]);
  831. }
  832.  
  833. extern __inline int
  834.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  835.     _mm_ucomilt_ss(__m128 __A, __m128 __B) {
  836.   return (__A[0] < __B[0]);
  837. }
  838.  
  839. extern __inline int
  840.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  841.     _mm_ucomile_ss(__m128 __A, __m128 __B) {
  842.   return (__A[0] <= __B[0]);
  843. }
  844.  
  845. extern __inline int
  846.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  847.     _mm_ucomigt_ss(__m128 __A, __m128 __B) {
  848.   return (__A[0] > __B[0]);
  849. }
  850.  
  851. extern __inline int
  852.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  853.     _mm_ucomige_ss(__m128 __A, __m128 __B) {
  854.   return (__A[0] >= __B[0]);
  855. }
  856.  
  857. extern __inline int
  858.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  859.     _mm_ucomineq_ss(__m128 __A, __m128 __B) {
  860.   return (__A[0] != __B[0]);
  861. }
  862.  
  863. extern __inline float
  864.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  865.     _mm_cvtss_f32(__m128 __A) {
  866.   return ((__v4sf)__A)[0];
  867. }
  868.  
  869. /* Convert the lower SPFP value to a 32-bit integer according to the current
  870.    rounding mode.  */
  871. extern __inline int
  872.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  873.     _mm_cvtss_si32(__m128 __A) {
  874.   int __res;
  875. #ifdef _ARCH_PWR8
  876.   double __dtmp;
  877.   __asm__(
  878. #ifdef __LITTLE_ENDIAN__
  879.       "xxsldwi %x0,%x0,%x0,3;\n"
  880. #endif
  881.       "xscvspdp %x2,%x0;\n"
  882.       "fctiw  %2,%2;\n"
  883.       "mfvsrd  %1,%x2;\n"
  884.       : "+wa"(__A), "=r"(__res), "=f"(__dtmp)
  885.       :);
  886. #else
  887.   __res = __builtin_rint(__A[0]);
  888. #endif
  889.   return __res;
  890. }
  891.  
  892. extern __inline int
  893.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  894.     _mm_cvt_ss2si(__m128 __A) {
  895.   return _mm_cvtss_si32(__A);
  896. }
  897.  
  898. /* Convert the lower SPFP value to a 32-bit integer according to the
  899.    current rounding mode.  */
  900.  
  901. /* Intel intrinsic.  */
  902. extern __inline long long
  903.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  904.     _mm_cvtss_si64(__m128 __A) {
  905.   long long __res;
  906. #if defined(_ARCH_PWR8) && defined(__powerpc64__)
  907.   double __dtmp;
  908.   __asm__(
  909. #ifdef __LITTLE_ENDIAN__
  910.       "xxsldwi %x0,%x0,%x0,3;\n"
  911. #endif
  912.       "xscvspdp %x2,%x0;\n"
  913.       "fctid  %2,%2;\n"
  914.       "mfvsrd  %1,%x2;\n"
  915.       : "+wa"(__A), "=r"(__res), "=f"(__dtmp)
  916.       :);
  917. #else
  918.   __res = __builtin_llrint(__A[0]);
  919. #endif
  920.   return __res;
  921. }
  922.  
  923. /* Microsoft intrinsic.  */
  924. extern __inline long long
  925.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  926.     _mm_cvtss_si64x(__m128 __A) {
  927.   return _mm_cvtss_si64((__v4sf)__A);
  928. }
  929.  
  930. /* Constants for use with _mm_prefetch.  */
  931. enum _mm_hint {
  932.   /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit.  */
  933.   _MM_HINT_ET0 = 7,
  934.   _MM_HINT_ET1 = 6,
  935.   _MM_HINT_T0 = 3,
  936.   _MM_HINT_T1 = 2,
  937.   _MM_HINT_T2 = 1,
  938.   _MM_HINT_NTA = 0
  939. };
  940.  
  941. /* Loads one cache line from address P to a location "closer" to the
  942.    processor.  The selector I specifies the type of prefetch operation.  */
  943. extern __inline void
  944.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  945.     _mm_prefetch(const void *__P, enum _mm_hint __I) {
  946.   /* Current PowerPC will ignores the hint parameters.  */
  947.   __builtin_prefetch(__P);
  948. }
  949.  
  950. /* Convert the two lower SPFP values to 32-bit integers according to the
  951.    current rounding mode.  Return the integers in packed form.  */
  952. extern __inline __m64
  953.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  954.     _mm_cvtps_pi32(__m128 __A) {
  955.   /* Splat two lower SPFP values to both halves.  */
  956.   __v4sf __temp, __rounded;
  957.   __vector unsigned long long __result;
  958.  
  959.   /* Splat two lower SPFP values to both halves.  */
  960.   __temp = (__v4sf)vec_splat((__vector long long)__A, 0);
  961.   __rounded = vec_rint(__temp);
  962.   __result = (__vector unsigned long long)vec_cts(__rounded, 0);
  963.  
  964.   return (__m64)((__vector long long)__result)[0];
  965. }
  966.  
  967. extern __inline __m64
  968.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  969.     _mm_cvt_ps2pi(__m128 __A) {
  970.   return _mm_cvtps_pi32(__A);
  971. }
  972.  
  973. /* Truncate the lower SPFP value to a 32-bit integer.  */
  974. extern __inline int
  975.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  976.     _mm_cvttss_si32(__m128 __A) {
  977.   /* Extract the lower float element.  */
  978.   float __temp = __A[0];
  979.   /* truncate to 32-bit integer and return.  */
  980.   return __temp;
  981. }
  982.  
  983. extern __inline int
  984.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  985.     _mm_cvtt_ss2si(__m128 __A) {
  986.   return _mm_cvttss_si32(__A);
  987. }
  988.  
  989. /* Intel intrinsic.  */
  990. extern __inline long long
  991.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  992.     _mm_cvttss_si64(__m128 __A) {
  993.   /* Extract the lower float element.  */
  994.   float __temp = __A[0];
  995.   /* truncate to 32-bit integer and return.  */
  996.   return __temp;
  997. }
  998.  
  999. /* Microsoft intrinsic.  */
  1000. extern __inline long long
  1001.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1002.     _mm_cvttss_si64x(__m128 __A) {
  1003.   /* Extract the lower float element.  */
  1004.   float __temp = __A[0];
  1005.   /* truncate to 32-bit integer and return.  */
  1006.   return __temp;
  1007. }
  1008.  
  1009. /* Truncate the two lower SPFP values to 32-bit integers.  Return the
  1010.    integers in packed form.  */
  1011. extern __inline __m64
  1012.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1013.     _mm_cvttps_pi32(__m128 __A) {
  1014.   __v4sf __temp;
  1015.   __vector unsigned long long __result;
  1016.  
  1017.   /* Splat two lower SPFP values to both halves.  */
  1018.   __temp = (__v4sf)vec_splat((__vector long long)__A, 0);
  1019.   __result = (__vector unsigned long long)vec_cts(__temp, 0);
  1020.  
  1021.   return (__m64)((__vector long long)__result)[0];
  1022. }
  1023.  
  1024. extern __inline __m64
  1025.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1026.     _mm_cvtt_ps2pi(__m128 __A) {
  1027.   return _mm_cvttps_pi32(__A);
  1028. }
  1029.  
  1030. /* Convert B to a SPFP value and insert it as element zero in A.  */
  1031. extern __inline __m128
  1032.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1033.     _mm_cvtsi32_ss(__m128 __A, int __B) {
  1034.   float __temp = __B;
  1035.   __A[0] = __temp;
  1036.  
  1037.   return __A;
  1038. }
  1039.  
  1040. extern __inline __m128
  1041.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1042.     _mm_cvt_si2ss(__m128 __A, int __B) {
  1043.   return _mm_cvtsi32_ss(__A, __B);
  1044. }
  1045.  
  1046. /* Convert B to a SPFP value and insert it as element zero in A.  */
  1047. /* Intel intrinsic.  */
  1048. extern __inline __m128
  1049.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1050.     _mm_cvtsi64_ss(__m128 __A, long long __B) {
  1051.   float __temp = __B;
  1052.   __A[0] = __temp;
  1053.  
  1054.   return __A;
  1055. }
  1056.  
  1057. /* Microsoft intrinsic.  */
  1058. extern __inline __m128
  1059.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1060.     _mm_cvtsi64x_ss(__m128 __A, long long __B) {
  1061.   return _mm_cvtsi64_ss(__A, __B);
  1062. }
  1063.  
  1064. /* Convert the two 32-bit values in B to SPFP form and insert them
  1065.    as the two lower elements in A.  */
  1066. extern __inline __m128
  1067.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1068.     _mm_cvtpi32_ps(__m128 __A, __m64 __B) {
  1069.   __vector signed int __vm1;
  1070.   __vector float __vf1;
  1071.  
  1072.   __vm1 = (__vector signed int)(__vector unsigned long long){__B, __B};
  1073.   __vf1 = (__vector float)vec_ctf(__vm1, 0);
  1074.  
  1075.   return ((__m128)(__vector unsigned long long){
  1076.       ((__vector unsigned long long)__vf1)[0],
  1077.       ((__vector unsigned long long)__A)[1]});
  1078. }
  1079.  
  1080. extern __inline __m128
  1081.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1082.     _mm_cvt_pi2ps(__m128 __A, __m64 __B) {
  1083.   return _mm_cvtpi32_ps(__A, __B);
  1084. }
  1085.  
  1086. /* Convert the four signed 16-bit values in A to SPFP form.  */
  1087. extern __inline __m128
  1088.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1089.     _mm_cvtpi16_ps(__m64 __A) {
  1090.   __vector signed short __vs8;
  1091.   __vector signed int __vi4;
  1092.   __vector float __vf1;
  1093.  
  1094.   __vs8 = (__vector signed short)(__vector unsigned long long){__A, __A};
  1095.   __vi4 = vec_vupklsh(__vs8);
  1096.   __vf1 = (__vector float)vec_ctf(__vi4, 0);
  1097.  
  1098.   return (__m128)__vf1;
  1099. }
  1100.  
  1101. /* Convert the four unsigned 16-bit values in A to SPFP form.  */
  1102. extern __inline __m128
  1103.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1104.     _mm_cvtpu16_ps(__m64 __A) {
  1105.   const __vector unsigned short __zero = {0, 0, 0, 0, 0, 0, 0, 0};
  1106.   __vector unsigned short __vs8;
  1107.   __vector unsigned int __vi4;
  1108.   __vector float __vf1;
  1109.  
  1110.   __vs8 = (__vector unsigned short)(__vector unsigned long long){__A, __A};
  1111.   __vi4 = (__vector unsigned int)vec_mergel
  1112. #ifdef __LITTLE_ENDIAN__
  1113.       (__vs8, __zero);
  1114. #else
  1115.       (__zero, __vs8);
  1116. #endif
  1117.   __vf1 = (__vector float)vec_ctf(__vi4, 0);
  1118.  
  1119.   return (__m128)__vf1;
  1120. }
  1121.  
  1122. /* Convert the low four signed 8-bit values in A to SPFP form.  */
  1123. extern __inline __m128
  1124.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1125.     _mm_cvtpi8_ps(__m64 __A) {
  1126.   __vector signed char __vc16;
  1127.   __vector signed short __vs8;
  1128.   __vector signed int __vi4;
  1129.   __vector float __vf1;
  1130.  
  1131.   __vc16 = (__vector signed char)(__vector unsigned long long){__A, __A};
  1132.   __vs8 = vec_vupkhsb(__vc16);
  1133.   __vi4 = vec_vupkhsh(__vs8);
  1134.   __vf1 = (__vector float)vec_ctf(__vi4, 0);
  1135.  
  1136.   return (__m128)__vf1;
  1137. }
  1138.  
  1139. /* Convert the low four unsigned 8-bit values in A to SPFP form.  */
  1140. extern __inline __m128
  1141.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1142.  
  1143.     _mm_cvtpu8_ps(__m64 __A) {
  1144.   const __vector unsigned char __zero = {0, 0, 0, 0, 0, 0, 0, 0};
  1145.   __vector unsigned char __vc16;
  1146.   __vector unsigned short __vs8;
  1147.   __vector unsigned int __vi4;
  1148.   __vector float __vf1;
  1149.  
  1150.   __vc16 = (__vector unsigned char)(__vector unsigned long long){__A, __A};
  1151. #ifdef __LITTLE_ENDIAN__
  1152.   __vs8 = (__vector unsigned short)vec_mergel(__vc16, __zero);
  1153.   __vi4 =
  1154.       (__vector unsigned int)vec_mergeh(__vs8, (__vector unsigned short)__zero);
  1155. #else
  1156.   __vs8 = (__vector unsigned short)vec_mergel(__zero, __vc16);
  1157.   __vi4 =
  1158.       (__vector unsigned int)vec_mergeh((__vector unsigned short)__zero, __vs8);
  1159. #endif
  1160.   __vf1 = (__vector float)vec_ctf(__vi4, 0);
  1161.  
  1162.   return (__m128)__vf1;
  1163. }
  1164.  
  1165. /* Convert the four signed 32-bit values in A and B to SPFP form.  */
  1166. extern __inline __m128
  1167.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1168.     _mm_cvtpi32x2_ps(__m64 __A, __m64 __B) {
  1169.   __vector signed int __vi4;
  1170.   __vector float __vf4;
  1171.  
  1172.   __vi4 = (__vector signed int)(__vector unsigned long long){__A, __B};
  1173.   __vf4 = (__vector float)vec_ctf(__vi4, 0);
  1174.   return (__m128)__vf4;
  1175. }
  1176.  
  1177. /* Convert the four SPFP values in A to four signed 16-bit integers.  */
  1178. extern __inline __m64
  1179.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1180.     _mm_cvtps_pi16(__m128 __A) {
  1181.   __v4sf __rounded;
  1182.   __vector signed int __temp;
  1183.   __vector unsigned long long __result;
  1184.  
  1185.   __rounded = vec_rint(__A);
  1186.   __temp = vec_cts(__rounded, 0);
  1187.   __result = (__vector unsigned long long)vec_pack(__temp, __temp);
  1188.  
  1189.   return (__m64)((__vector long long)__result)[0];
  1190. }
  1191.  
  1192. /* Convert the four SPFP values in A to four signed 8-bit integers.  */
  1193. extern __inline __m64
  1194.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1195.     _mm_cvtps_pi8(__m128 __A) {
  1196.   __v4sf __rounded;
  1197.   __vector signed int __tmp_i;
  1198.   static const __vector signed int __zero = {0, 0, 0, 0};
  1199.   __vector signed short __tmp_s;
  1200.   __vector signed char __res_v;
  1201.  
  1202.   __rounded = vec_rint(__A);
  1203.   __tmp_i = vec_cts(__rounded, 0);
  1204.   __tmp_s = vec_pack(__tmp_i, __zero);
  1205.   __res_v = vec_pack(__tmp_s, __tmp_s);
  1206.   return (__m64)((__vector long long)__res_v)[0];
  1207. }
  1208.  
  1209. /* Selects four specific SPFP values from A and B based on MASK.  */
  1210. extern __inline __m128
  1211.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1212.  
  1213.     _mm_shuffle_ps(__m128 __A, __m128 __B, int const __mask) {
  1214.   unsigned long __element_selector_10 = __mask & 0x03;
  1215.   unsigned long __element_selector_32 = (__mask >> 2) & 0x03;
  1216.   unsigned long __element_selector_54 = (__mask >> 4) & 0x03;
  1217.   unsigned long __element_selector_76 = (__mask >> 6) & 0x03;
  1218.   static const unsigned int __permute_selectors[4] = {
  1219. #ifdef __LITTLE_ENDIAN__
  1220.       0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
  1221. #else
  1222.       0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F
  1223. #endif
  1224.   };
  1225.   __vector unsigned int __t;
  1226.  
  1227.   __t[0] = __permute_selectors[__element_selector_10];
  1228.   __t[1] = __permute_selectors[__element_selector_32];
  1229.   __t[2] = __permute_selectors[__element_selector_54] + 0x10101010;
  1230.   __t[3] = __permute_selectors[__element_selector_76] + 0x10101010;
  1231.   return vec_perm((__v4sf)__A, (__v4sf)__B, (__vector unsigned char)__t);
  1232. }
  1233.  
  1234. /* Selects and interleaves the upper two SPFP values from A and B.  */
  1235. extern __inline __m128
  1236.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1237.     _mm_unpackhi_ps(__m128 __A, __m128 __B) {
  1238.   return (__m128)vec_vmrglw((__v4sf)__A, (__v4sf)__B);
  1239. }
  1240.  
  1241. /* Selects and interleaves the lower two SPFP values from A and B.  */
  1242. extern __inline __m128
  1243.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1244.     _mm_unpacklo_ps(__m128 __A, __m128 __B) {
  1245.   return (__m128)vec_vmrghw((__v4sf)__A, (__v4sf)__B);
  1246. }
  1247.  
  1248. /* Sets the upper two SPFP values with 64-bits of data loaded from P;
  1249.    the lower two values are passed through from A.  */
  1250. extern __inline __m128
  1251.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1252.     _mm_loadh_pi(__m128 __A, __m64 const *__P) {
  1253.   __vector unsigned long long __a = (__vector unsigned long long)__A;
  1254.   __vector unsigned long long __p = vec_splats(*__P);
  1255.   __a[1] = __p[1];
  1256.  
  1257.   return (__m128)__a;
  1258. }
  1259.  
  1260. /* Stores the upper two SPFP values of A into P.  */
  1261. extern __inline void
  1262.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1263.     _mm_storeh_pi(__m64 *__P, __m128 __A) {
  1264.   __vector unsigned long long __a = (__vector unsigned long long)__A;
  1265.  
  1266.   *__P = __a[1];
  1267. }
  1268.  
  1269. /* Moves the upper two values of B into the lower two values of A.  */
  1270. extern __inline __m128
  1271.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1272.     _mm_movehl_ps(__m128 __A, __m128 __B) {
  1273.   return (__m128)vec_mergel((__vector unsigned long long)__B,
  1274.                             (__vector unsigned long long)__A);
  1275. }
  1276.  
  1277. /* Moves the lower two values of B into the upper two values of A.  */
  1278. extern __inline __m128
  1279.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1280.     _mm_movelh_ps(__m128 __A, __m128 __B) {
  1281.   return (__m128)vec_mergeh((__vector unsigned long long)__A,
  1282.                             (__vector unsigned long long)__B);
  1283. }
  1284.  
  1285. /* Sets the lower two SPFP values with 64-bits of data loaded from P;
  1286.    the upper two values are passed through from A.  */
  1287. extern __inline __m128
  1288.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1289.     _mm_loadl_pi(__m128 __A, __m64 const *__P) {
  1290.   __vector unsigned long long __a = (__vector unsigned long long)__A;
  1291.   __vector unsigned long long __p = vec_splats(*__P);
  1292.   __a[0] = __p[0];
  1293.  
  1294.   return (__m128)__a;
  1295. }
  1296.  
  1297. /* Stores the lower two SPFP values of A into P.  */
  1298. extern __inline void
  1299.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1300.     _mm_storel_pi(__m64 *__P, __m128 __A) {
  1301.   __vector unsigned long long __a = (__vector unsigned long long)__A;
  1302.  
  1303.   *__P = __a[0];
  1304. }
  1305.  
  1306. #ifdef _ARCH_PWR8
  1307. /* Intrinsic functions that require PowerISA 2.07 minimum.  */
  1308.  
  1309. /* Creates a 4-bit mask from the most significant bits of the SPFP values.  */
  1310. extern __inline int
  1311.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1312.     _mm_movemask_ps(__m128 __A) {
  1313. #ifdef _ARCH_PWR10
  1314.   return vec_extractm((__vector unsigned int)__A);
  1315. #else
  1316.   __vector unsigned long long __result;
  1317.   static const __vector unsigned int __perm_mask = {
  1318. #ifdef __LITTLE_ENDIAN__
  1319.       0x00204060, 0x80808080, 0x80808080, 0x80808080
  1320. #else
  1321.       0x80808080, 0x80808080, 0x80808080, 0x00204060
  1322. #endif
  1323.   };
  1324.  
  1325.   __result = ((__vector unsigned long long)vec_vbpermq(
  1326.       (__vector unsigned char)__A, (__vector unsigned char)__perm_mask));
  1327.  
  1328. #ifdef __LITTLE_ENDIAN__
  1329.   return __result[1];
  1330. #else
  1331.   return __result[0];
  1332. #endif
  1333. #endif /* !_ARCH_PWR10 */
  1334. }
  1335. #endif /* _ARCH_PWR8 */
  1336.  
  1337. /* Create a vector with all four elements equal to *P.  */
  1338. extern __inline __m128
  1339.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1340.     _mm_load1_ps(float const *__P) {
  1341.   return _mm_set1_ps(*__P);
  1342. }
  1343.  
  1344. extern __inline __m128
  1345.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1346.     _mm_load_ps1(float const *__P) {
  1347.   return _mm_load1_ps(__P);
  1348. }
  1349.  
  1350. /* Extracts one of the four words of A.  The selector N must be immediate.  */
  1351. extern __inline int
  1352.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1353.     _mm_extract_pi16(__m64 const __A, int const __N) {
  1354.   unsigned int __shiftr = __N & 3;
  1355. #ifdef __BIG_ENDIAN__
  1356.   __shiftr = 3 - __shiftr;
  1357. #endif
  1358.  
  1359.   return ((__A >> (__shiftr * 16)) & 0xffff);
  1360. }
  1361.  
  1362. extern __inline int
  1363.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1364.     _m_pextrw(__m64 const __A, int const __N) {
  1365.   return _mm_extract_pi16(__A, __N);
  1366. }
  1367.  
  1368. /* Inserts word D into one of four words of A.  The selector N must be
  1369.    immediate.  */
  1370. extern __inline __m64
  1371.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1372.     _mm_insert_pi16(__m64 const __A, int const __D, int const __N) {
  1373.   const int __shiftl = (__N & 3) * 16;
  1374.   const __m64 __shiftD = (const __m64)__D << __shiftl;
  1375.   const __m64 __mask = 0xffffUL << __shiftl;
  1376.   __m64 __result = (__A & (~__mask)) | (__shiftD & __mask);
  1377.  
  1378.   return __result;
  1379. }
  1380.  
  1381. extern __inline __m64
  1382.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1383.     _m_pinsrw(__m64 const __A, int const __D, int const __N) {
  1384.   return _mm_insert_pi16(__A, __D, __N);
  1385. }
  1386.  
  1387. /* Compute the element-wise maximum of signed 16-bit values.  */
  1388. extern __inline __m64
  1389.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1390.  
  1391.     _mm_max_pi16(__m64 __A, __m64 __B) {
  1392. #if _ARCH_PWR8
  1393.   __vector signed short __a, __b, __r;
  1394.   __vector __bool short __c;
  1395.  
  1396.   __a = (__vector signed short)vec_splats(__A);
  1397.   __b = (__vector signed short)vec_splats(__B);
  1398.   __c = (__vector __bool short)vec_cmpgt(__a, __b);
  1399.   __r = vec_sel(__b, __a, __c);
  1400.   return (__m64)((__vector long long)__r)[0];
  1401. #else
  1402.   __m64_union __m1, __m2, __res;
  1403.  
  1404.   __m1.as_m64 = __A;
  1405.   __m2.as_m64 = __B;
  1406.  
  1407.   __res.as_short[0] = (__m1.as_short[0] > __m2.as_short[0]) ? __m1.as_short[0]
  1408.                                                             : __m2.as_short[0];
  1409.   __res.as_short[1] = (__m1.as_short[1] > __m2.as_short[1]) ? __m1.as_short[1]
  1410.                                                             : __m2.as_short[1];
  1411.   __res.as_short[2] = (__m1.as_short[2] > __m2.as_short[2]) ? __m1.as_short[2]
  1412.                                                             : __m2.as_short[2];
  1413.   __res.as_short[3] = (__m1.as_short[3] > __m2.as_short[3]) ? __m1.as_short[3]
  1414.                                                             : __m2.as_short[3];
  1415.  
  1416.   return (__m64)__res.as_m64;
  1417. #endif
  1418. }
  1419.  
  1420. extern __inline __m64
  1421.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1422.     _m_pmaxsw(__m64 __A, __m64 __B) {
  1423.   return _mm_max_pi16(__A, __B);
  1424. }
  1425.  
  1426. /* Compute the element-wise maximum of unsigned 8-bit values.  */
  1427. extern __inline __m64
  1428.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1429.     _mm_max_pu8(__m64 __A, __m64 __B) {
  1430. #if _ARCH_PWR8
  1431.   __vector unsigned char __a, __b, __r;
  1432.   __vector __bool char __c;
  1433.  
  1434.   __a = (__vector unsigned char)vec_splats(__A);
  1435.   __b = (__vector unsigned char)vec_splats(__B);
  1436.   __c = (__vector __bool char)vec_cmpgt(__a, __b);
  1437.   __r = vec_sel(__b, __a, __c);
  1438.   return (__m64)((__vector long long)__r)[0];
  1439. #else
  1440.   __m64_union __m1, __m2, __res;
  1441.   long __i;
  1442.  
  1443.   __m1.as_m64 = __A;
  1444.   __m2.as_m64 = __B;
  1445.  
  1446.   for (__i = 0; __i < 8; __i++)
  1447.     __res.as_char[__i] =
  1448.         ((unsigned char)__m1.as_char[__i] > (unsigned char)__m2.as_char[__i])
  1449.             ? __m1.as_char[__i]
  1450.             : __m2.as_char[__i];
  1451.  
  1452.   return (__m64)__res.as_m64;
  1453. #endif
  1454. }
  1455.  
  1456. extern __inline __m64
  1457.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1458.     _m_pmaxub(__m64 __A, __m64 __B) {
  1459.   return _mm_max_pu8(__A, __B);
  1460. }
  1461.  
  1462. /* Compute the element-wise minimum of signed 16-bit values.  */
  1463. extern __inline __m64
  1464.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1465.     _mm_min_pi16(__m64 __A, __m64 __B) {
  1466. #if _ARCH_PWR8
  1467.   __vector signed short __a, __b, __r;
  1468.   __vector __bool short __c;
  1469.  
  1470.   __a = (__vector signed short)vec_splats(__A);
  1471.   __b = (__vector signed short)vec_splats(__B);
  1472.   __c = (__vector __bool short)vec_cmplt(__a, __b);
  1473.   __r = vec_sel(__b, __a, __c);
  1474.   return (__m64)((__vector long long)__r)[0];
  1475. #else
  1476.   __m64_union __m1, __m2, __res;
  1477.  
  1478.   __m1.as_m64 = __A;
  1479.   __m2.as_m64 = __B;
  1480.  
  1481.   __res.as_short[0] = (__m1.as_short[0] < __m2.as_short[0]) ? __m1.as_short[0]
  1482.                                                             : __m2.as_short[0];
  1483.   __res.as_short[1] = (__m1.as_short[1] < __m2.as_short[1]) ? __m1.as_short[1]
  1484.                                                             : __m2.as_short[1];
  1485.   __res.as_short[2] = (__m1.as_short[2] < __m2.as_short[2]) ? __m1.as_short[2]
  1486.                                                             : __m2.as_short[2];
  1487.   __res.as_short[3] = (__m1.as_short[3] < __m2.as_short[3]) ? __m1.as_short[3]
  1488.                                                             : __m2.as_short[3];
  1489.  
  1490.   return (__m64)__res.as_m64;
  1491. #endif
  1492. }
  1493.  
  1494. extern __inline __m64
  1495.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1496.     _m_pminsw(__m64 __A, __m64 __B) {
  1497.   return _mm_min_pi16(__A, __B);
  1498. }
  1499.  
  1500. /* Compute the element-wise minimum of unsigned 8-bit values.  */
  1501. extern __inline __m64
  1502.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1503.     _mm_min_pu8(__m64 __A, __m64 __B) {
  1504. #if _ARCH_PWR8
  1505.   __vector unsigned char __a, __b, __r;
  1506.   __vector __bool char __c;
  1507.  
  1508.   __a = (__vector unsigned char)vec_splats(__A);
  1509.   __b = (__vector unsigned char)vec_splats(__B);
  1510.   __c = (__vector __bool char)vec_cmplt(__a, __b);
  1511.   __r = vec_sel(__b, __a, __c);
  1512.   return (__m64)((__vector long long)__r)[0];
  1513. #else
  1514.   __m64_union __m1, __m2, __res;
  1515.   long __i;
  1516.  
  1517.   __m1.as_m64 = __A;
  1518.   __m2.as_m64 = __B;
  1519.  
  1520.   for (__i = 0; __i < 8; __i++)
  1521.     __res.as_char[__i] =
  1522.         ((unsigned char)__m1.as_char[__i] < (unsigned char)__m2.as_char[__i])
  1523.             ? __m1.as_char[__i]
  1524.             : __m2.as_char[__i];
  1525.  
  1526.   return (__m64)__res.as_m64;
  1527. #endif
  1528. }
  1529.  
  1530. extern __inline __m64
  1531.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1532.     _m_pminub(__m64 __A, __m64 __B) {
  1533.   return _mm_min_pu8(__A, __B);
  1534. }
  1535.  
  1536. /* Create an 8-bit mask of the signs of 8-bit values.  */
  1537. extern __inline int
  1538.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1539.     _mm_movemask_pi8(__m64 __A) {
  1540. #ifdef __powerpc64__
  1541.   unsigned long long __p =
  1542. #ifdef __LITTLE_ENDIAN__
  1543.       0x0008101820283038UL; // permute control for sign bits
  1544. #else
  1545.       0x3830282018100800UL; // permute control for sign bits
  1546. #endif
  1547.   return __builtin_bpermd(__p, __A);
  1548. #else
  1549. #ifdef __LITTLE_ENDIAN__
  1550.   unsigned int __mask = 0x20283038UL;
  1551.   unsigned int __r1 = __builtin_bpermd(__mask, __A) & 0xf;
  1552.   unsigned int __r2 = __builtin_bpermd(__mask, __A >> 32) & 0xf;
  1553. #else
  1554.   unsigned int __mask = 0x38302820UL;
  1555.   unsigned int __r1 = __builtin_bpermd(__mask, __A >> 32) & 0xf;
  1556.   unsigned int __r2 = __builtin_bpermd(__mask, __A) & 0xf;
  1557. #endif
  1558.   return (__r2 << 4) | __r1;
  1559. #endif
  1560. }
  1561.  
  1562. extern __inline int
  1563.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1564.     _m_pmovmskb(__m64 __A) {
  1565.   return _mm_movemask_pi8(__A);
  1566. }
  1567.  
  1568. /* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
  1569.    in B and produce the high 16 bits of the 32-bit results.  */
  1570. extern __inline __m64
  1571.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1572.     _mm_mulhi_pu16(__m64 __A, __m64 __B) {
  1573.   __vector unsigned short __a, __b;
  1574.   __vector unsigned short __c;
  1575.   __vector unsigned int __w0, __w1;
  1576.   __vector unsigned char __xform1 = {
  1577. #ifdef __LITTLE_ENDIAN__
  1578.       0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,
  1579.       0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
  1580. #else
  1581.       0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x00,
  1582.       0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15
  1583. #endif
  1584.   };
  1585.  
  1586.   __a = (__vector unsigned short)vec_splats(__A);
  1587.   __b = (__vector unsigned short)vec_splats(__B);
  1588.  
  1589.   __w0 = vec_vmuleuh(__a, __b);
  1590.   __w1 = vec_vmulouh(__a, __b);
  1591.   __c = (__vector unsigned short)vec_perm(__w0, __w1, __xform1);
  1592.  
  1593.   return (__m64)((__vector long long)__c)[0];
  1594. }
  1595.  
  1596. extern __inline __m64
  1597.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1598.     _m_pmulhuw(__m64 __A, __m64 __B) {
  1599.   return _mm_mulhi_pu16(__A, __B);
  1600. }
  1601.  
  1602. /* Return a combination of the four 16-bit values in A.  The selector
  1603.    must be an immediate.  */
  1604. extern __inline __m64
  1605.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1606.     _mm_shuffle_pi16(__m64 __A, int const __N) {
  1607.   unsigned long __element_selector_10 = __N & 0x03;
  1608.   unsigned long __element_selector_32 = (__N >> 2) & 0x03;
  1609.   unsigned long __element_selector_54 = (__N >> 4) & 0x03;
  1610.   unsigned long __element_selector_76 = (__N >> 6) & 0x03;
  1611.   static const unsigned short __permute_selectors[4] = {
  1612. #ifdef __LITTLE_ENDIAN__
  1613.       0x0908, 0x0B0A, 0x0D0C, 0x0F0E
  1614. #else
  1615.       0x0607, 0x0405, 0x0203, 0x0001
  1616. #endif
  1617.   };
  1618.   __m64_union __t;
  1619.   __vector unsigned long long __a, __p, __r;
  1620.  
  1621. #ifdef __LITTLE_ENDIAN__
  1622.   __t.as_short[0] = __permute_selectors[__element_selector_10];
  1623.   __t.as_short[1] = __permute_selectors[__element_selector_32];
  1624.   __t.as_short[2] = __permute_selectors[__element_selector_54];
  1625.   __t.as_short[3] = __permute_selectors[__element_selector_76];
  1626. #else
  1627.   __t.as_short[3] = __permute_selectors[__element_selector_10];
  1628.   __t.as_short[2] = __permute_selectors[__element_selector_32];
  1629.   __t.as_short[1] = __permute_selectors[__element_selector_54];
  1630.   __t.as_short[0] = __permute_selectors[__element_selector_76];
  1631. #endif
  1632.   __p = vec_splats(__t.as_m64);
  1633.   __a = vec_splats(__A);
  1634.   __r = vec_perm(__a, __a, (__vector unsigned char)__p);
  1635.   return (__m64)((__vector long long)__r)[0];
  1636. }
  1637.  
  1638. extern __inline __m64
  1639.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1640.     _m_pshufw(__m64 __A, int const __N) {
  1641.   return _mm_shuffle_pi16(__A, __N);
  1642. }
  1643.  
  1644. /* Conditionally store byte elements of A into P.  The high bit of each
  1645.    byte in the selector N determines whether the corresponding byte from
  1646.    A is stored.  */
  1647. extern __inline void
  1648.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1649.     _mm_maskmove_si64(__m64 __A, __m64 __N, char *__P) {
  1650.   __m64 __hibit = 0x8080808080808080UL;
  1651.   __m64 __mask, __tmp;
  1652.   __m64 *__p = (__m64 *)__P;
  1653.  
  1654.   __tmp = *__p;
  1655.   __mask = _mm_cmpeq_pi8((__N & __hibit), __hibit);
  1656.   __tmp = (__tmp & (~__mask)) | (__A & __mask);
  1657.   *__p = __tmp;
  1658. }
  1659.  
  1660. extern __inline void
  1661.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1662.     _m_maskmovq(__m64 __A, __m64 __N, char *__P) {
  1663.   _mm_maskmove_si64(__A, __N, __P);
  1664. }
  1665.  
  1666. /* Compute the rounded averages of the unsigned 8-bit values in A and B.  */
  1667. extern __inline __m64
  1668.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1669.     _mm_avg_pu8(__m64 __A, __m64 __B) {
  1670.   __vector unsigned char __a, __b, __c;
  1671.  
  1672.   __a = (__vector unsigned char)vec_splats(__A);
  1673.   __b = (__vector unsigned char)vec_splats(__B);
  1674.   __c = vec_avg(__a, __b);
  1675.   return (__m64)((__vector long long)__c)[0];
  1676. }
  1677.  
  1678. extern __inline __m64
  1679.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1680.     _m_pavgb(__m64 __A, __m64 __B) {
  1681.   return _mm_avg_pu8(__A, __B);
  1682. }
  1683.  
  1684. /* Compute the rounded averages of the unsigned 16-bit values in A and B.  */
  1685. extern __inline __m64
  1686.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1687.     _mm_avg_pu16(__m64 __A, __m64 __B) {
  1688.   __vector unsigned short __a, __b, __c;
  1689.  
  1690.   __a = (__vector unsigned short)vec_splats(__A);
  1691.   __b = (__vector unsigned short)vec_splats(__B);
  1692.   __c = vec_avg(__a, __b);
  1693.   return (__m64)((__vector long long)__c)[0];
  1694. }
  1695.  
  1696. extern __inline __m64
  1697.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1698.     _m_pavgw(__m64 __A, __m64 __B) {
  1699.   return _mm_avg_pu16(__A, __B);
  1700. }
  1701.  
  1702. /* Compute the sum of the absolute differences of the unsigned 8-bit
  1703.    values in A and B.  Return the value in the lower 16-bit word; the
  1704.    upper words are cleared.  */
  1705. extern __inline __m64
  1706.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1707.     _mm_sad_pu8(__m64 __A, __m64 __B) {
  1708.   __vector unsigned char __a, __b;
  1709.   __vector unsigned char __vmin, __vmax, __vabsdiff;
  1710.   __vector signed int __vsum;
  1711.   const __vector unsigned int __zero = {0, 0, 0, 0};
  1712.   __m64_union __result = {0};
  1713.  
  1714.   __a = (__vector unsigned char)(__vector unsigned long long){0UL, __A};
  1715.   __b = (__vector unsigned char)(__vector unsigned long long){0UL, __B};
  1716.   __vmin = vec_min(__a, __b);
  1717.   __vmax = vec_max(__a, __b);
  1718.   __vabsdiff = vec_sub(__vmax, __vmin);
  1719.   /* Sum four groups of bytes into integers.  */
  1720.   __vsum = (__vector signed int)vec_sum4s(__vabsdiff, __zero);
  1721.   /* Sum across four integers with integer result.  */
  1722.   __vsum = vec_sums(__vsum, (__vector signed int)__zero);
  1723.   /* The sum is in the right most 32-bits of the vector result.
  1724.      Transfer to a GPR and truncate to 16 bits.  */
  1725.   __result.as_short[0] = __vsum[3];
  1726.   return __result.as_m64;
  1727. }
  1728.  
  1729. extern __inline __m64
  1730.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1731.     _m_psadbw(__m64 __A, __m64 __B) {
  1732.   return _mm_sad_pu8(__A, __B);
  1733. }
  1734.  
  1735. /* Stores the data in A to the address P without polluting the caches.  */
  1736. extern __inline void
  1737.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1738.     _mm_stream_pi(__m64 *__P, __m64 __A) {
  1739.   /* Use the data cache block touch for store transient.  */
  1740.   __asm__("     dcbtstt 0,%0" : : "b"(__P) : "memory");
  1741.   *__P = __A;
  1742. }
  1743.  
  1744. /* Likewise.  The address must be 16-byte aligned.  */
  1745. extern __inline void
  1746.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1747.     _mm_stream_ps(float *__P, __m128 __A) {
  1748.   /* Use the data cache block touch for store transient.  */
  1749.   __asm__("     dcbtstt 0,%0" : : "b"(__P) : "memory");
  1750.   _mm_store_ps(__P, __A);
  1751. }
  1752.  
  1753. /* Guarantees that every preceding store is globally visible before
  1754.    any subsequent store.  */
  1755. extern __inline void
  1756.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1757.     _mm_sfence(void) {
  1758.   /* Generate a light weight sync.  */
  1759.   __atomic_thread_fence(__ATOMIC_RELEASE);
  1760. }
  1761.  
  1762. /* The execution of the next instruction is delayed by an implementation
  1763.    specific amount of time.  The instruction does not modify the
  1764.    architectural state.  This is after the pop_options pragma because
  1765.    it does not require SSE support in the processor--the encoding is a
  1766.    nop on processors that do not support it.  */
  1767. extern __inline void
  1768.     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  1769.     _mm_pause(void) {
  1770.   /* There is no exact match with this construct, but the following is
  1771.      close to the desired effect.  */
  1772. #if _ARCH_PWR8
  1773.   /* On power8 and later processors we can depend on Program Priority
  1774.      (PRI) and associated "very low" PPI setting.  Since we don't know
  1775.      what PPI this thread is running at we: 1) save the current PRI
  1776.      from the PPR SPR into a local GRP, 2) set the PRI to "very low*
  1777.      via the special or 31,31,31 encoding. 3) issue an "isync" to
  1778.      insure the PRI change takes effect before we execute any more
  1779.      instructions.
  1780.      Now we can execute a lwsync (release barrier) while we execute
  1781.      this thread at "very low" PRI.  Finally we restore the original
  1782.      PRI and continue execution.  */
  1783.   unsigned long __PPR;
  1784.  
  1785.   __asm__ volatile("    mfppr   %0;"
  1786.                    "   or 31,31,31;"
  1787.                    "   isync;"
  1788.                    "   lwsync;"
  1789.                    "   isync;"
  1790.                    "   mtppr    %0;"
  1791.                    : "=r"(__PPR)
  1792.                    :
  1793.                    : "memory");
  1794. #else
  1795.   /* For older processor where we may not even have Program Priority
  1796.      controls we can only depend on Heavy Weight Sync.  */
  1797.   __atomic_thread_fence(__ATOMIC_SEQ_CST);
  1798. #endif
  1799. }
  1800.  
  1801. /* Transpose the 4x4 matrix composed of row[0-3].  */
  1802. #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3)                              \
  1803.   do {                                                                         \
  1804.     __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3);         \
  1805.     __v4sf __t0 = vec_vmrghw(__r0, __r1);                                      \
  1806.     __v4sf __t1 = vec_vmrghw(__r2, __r3);                                      \
  1807.     __v4sf __t2 = vec_vmrglw(__r0, __r1);                                      \
  1808.     __v4sf __t3 = vec_vmrglw(__r2, __r3);                                      \
  1809.     (row0) = (__v4sf)vec_mergeh((__vector long long)__t0,                      \
  1810.                                 (__vector long long)__t1);                     \
  1811.     (row1) = (__v4sf)vec_mergel((__vector long long)__t0,                      \
  1812.                                 (__vector long long)__t1);                     \
  1813.     (row2) = (__v4sf)vec_mergeh((__vector long long)__t2,                      \
  1814.                                 (__vector long long)__t3);                     \
  1815.     (row3) = (__v4sf)vec_mergel((__vector long long)__t2,                      \
  1816.                                 (__vector long long)__t3);                     \
  1817.   } while (0)
  1818.  
  1819. /* For backward source compatibility.  */
  1820. //# include <emmintrin.h>
  1821.  
  1822. #else
  1823. #include_next <xmmintrin.h>
  1824. #endif /* defined(__powerpc64__) &&                                            \
  1825.         *   (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) */
  1826.  
  1827. #endif /* XMMINTRIN_H_ */
  1828.