🔐 Sid Gifari File Manager Pro
v8.0.5 | 2026-06-14 13:34:58 | PHP 8.1.34
📂
/ (Root)
/
usr
/
include
/
c++
/
11
/
experimental
/
bits
📍 /usr/include/c++/11/experimental/bits
🔄 Refresh
✏️
Editing: simd_builtin.h
Read Only
// Simd Abi specific implementations -*- C++ -*- // Copyright (C) 2020-2021 Free Software Foundation, Inc. // // This file is part of the GNU ISO C++ Library. This library is free // software; you can redistribute it and/or modify it under the // terms of the GNU General Public License as published by the // Free Software Foundation; either version 3, or (at your option) // any later version. // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // Under Section 7 of GPL version 3, you are granted additional // permissions described in the GCC Runtime Library Exception, version // 3.1, as published by the Free Software Foundation. // You should have received a copy of the GNU General Public License and // a copy of the GCC Runtime Library Exception along with this program; // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see // <http://www.gnu.org/licenses/>. #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_ #define _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_ #if __cplusplus >= 201703L #include <array> #include <cmath> #include <cstdlib> _GLIBCXX_SIMD_BEGIN_NAMESPACE // _S_allbits{{{ template <typename _V> static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_allbits = reinterpret_cast<_V>(~__vector_type_t<char, sizeof(_V) / sizeof(char)>()); // }}} // _S_signmask, _S_absmask{{{ template <typename _V, typename = _VectorTraits<_V>> static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_signmask = __xor(_V() + 1, _V() - 1); template <typename _V, typename = _VectorTraits<_V>> static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_absmask = __andnot(_S_signmask<_V>, _S_allbits<_V>); //}}} // __vector_permute<Indices...>{{{ // Index == -1 requests zeroing of the output element template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>> constexpr _Tp __vector_permute(_Tp __x) { static_assert(sizeof...(_Indices) == _TVT::_S_full_size); return __make_vector<typename _TVT::value_type>( (_Indices == -1 ? 0 : __x[_Indices == -1 ? 0 : _Indices])...); } // }}} // __vector_shuffle<Indices...>{{{ // Index == -1 requests zeroing of the output element template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>> constexpr _Tp __vector_shuffle(_Tp __x, _Tp __y) { return _Tp{(_Indices == -1 ? 0 : _Indices < _TVT::_S_full_size ? __x[_Indices] : __y[_Indices - _TVT::_S_full_size])...}; } // }}} // __make_wrapper{{{ template <typename _Tp, typename... _Args> _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, sizeof...(_Args)> __make_wrapper(const _Args&... __args) { return __make_vector<_Tp>(__args...); } // }}} // __wrapper_bitcast{{{ template <typename _Tp, size_t _ToN = 0, typename _Up, size_t _M, size_t _Np = _ToN != 0 ? _ToN : sizeof(_Up) * _M / sizeof(_Tp)> _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _Np> __wrapper_bitcast(_SimdWrapper<_Up, _M> __x) { static_assert(_Np > 1); return __intrin_bitcast<__vector_type_t<_Tp, _Np>>(__x._M_data); } // }}} // __shift_elements_right{{{ // if (__shift % 2ⁿ == 0) => the low n Bytes are correct template <unsigned __shift, typename _Tp, typename _TVT = _VectorTraits<_Tp>> _GLIBCXX_SIMD_INTRINSIC _Tp __shift_elements_right(_Tp __v) { [[maybe_unused]] const auto __iv = __to_intrin(__v); static_assert(__shift <= sizeof(_Tp)); if constexpr (__shift == 0) return __v; else if constexpr (__shift == sizeof(_Tp)) return _Tp(); #if _GLIBCXX_SIMD_X86INTRIN // {{{ else if constexpr (__have_sse && __shift == 8 && _TVT::template _S_is<float, 4>) return _mm_movehl_ps(__iv, __iv); else if constexpr (__have_sse2 && __shift == 8 && _TVT::template _S_is<double, 2>) return _mm_unpackhi_pd(__iv, __iv); else if constexpr (__have_sse2 && sizeof(_Tp) == 16) return reinterpret_cast<typename _TVT::type>( _mm_srli_si128(reinterpret_cast<__m128i>(__iv), __shift)); else if constexpr (__shift == 16 && sizeof(_Tp) == 32) { /*if constexpr (__have_avx && _TVT::template _S_is<double, 4>) return _mm256_permute2f128_pd(__iv, __iv, 0x81); else if constexpr (__have_avx && _TVT::template _S_is<float, 8>) return _mm256_permute2f128_ps(__iv, __iv, 0x81); else if constexpr (__have_avx) return reinterpret_cast<typename _TVT::type>( _mm256_permute2f128_si256(__iv, __iv, 0x81)); else*/ return __zero_extend(__hi128(__v)); } else if constexpr (__have_avx2 && sizeof(_Tp) == 32 && __shift < 16) { const auto __vll = __vector_bitcast<_LLong>(__v); return reinterpret_cast<typename _TVT::type>( _mm256_alignr_epi8(_mm256_permute2x128_si256(__vll, __vll, 0x81), __vll, __shift)); } else if constexpr (__have_avx && sizeof(_Tp) == 32 && __shift < 16) { const auto __vll = __vector_bitcast<_LLong>(__v); return reinterpret_cast<typename _TVT::type>( __concat(_mm_alignr_epi8(__hi128(__vll), __lo128(__vll), __shift), _mm_srli_si128(__hi128(__vll), __shift))); } else if constexpr (sizeof(_Tp) == 32 && __shift > 16) return __zero_extend(__shift_elements_right<__shift - 16>(__hi128(__v))); else if constexpr (sizeof(_Tp) == 64 && __shift == 32) return __zero_extend(__hi256(__v)); else if constexpr (__have_avx512f && sizeof(_Tp) == 64) { if constexpr (__shift >= 48) return __zero_extend( __shift_elements_right<__shift - 48>(__extract<3, 4>(__v))); else if constexpr (__shift >= 32) return __zero_extend( __shift_elements_right<__shift - 32>(__hi256(__v))); else if constexpr (__shift % 8 == 0) return reinterpret_cast<typename _TVT::type>( _mm512_alignr_epi64(__m512i(), __intrin_bitcast<__m512i>(__v), __shift / 8)); else if constexpr (__shift % 4 == 0) return reinterpret_cast<typename _TVT::type>( _mm512_alignr_epi32(__m512i(), __intrin_bitcast<__m512i>(__v), __shift / 4)); else if constexpr (__have_avx512bw && __shift < 16) { const auto __vll = __vector_bitcast<_LLong>(__v); return reinterpret_cast<typename _TVT::type>( _mm512_alignr_epi8(_mm512_shuffle_i32x4(__vll, __vll, 0xf9), __vll, __shift)); } else if constexpr (__have_avx512bw && __shift < 32) { const auto __vll = __vector_bitcast<_LLong>(__v); return reinterpret_cast<typename _TVT::type>( _mm512_alignr_epi8(_mm512_shuffle_i32x4(__vll, __m512i(), 0xee), _mm512_shuffle_i32x4(__vll, __vll, 0xf9), __shift - 16)); } else __assert_unreachable<_Tp>(); } /* } else if constexpr (__shift % 16 == 0 && sizeof(_Tp) == 64) return __auto_bitcast(__extract<__shift / 16, 4>(__v)); */ #endif // _GLIBCXX_SIMD_X86INTRIN }}} else { constexpr int __chunksize = __shift % 8 == 0 ? 8 : __shift % 4 == 0 ? 4 : __shift % 2 == 0 ? 2 : 1; auto __w = __vector_bitcast<__int_with_sizeof_t<__chunksize>>(__v); using _Up = decltype(__w); return __intrin_bitcast<_Tp>( __call_with_n_evaluations<(sizeof(_Tp) - __shift) / __chunksize>( [](auto... __chunks) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return _Up{__chunks...}; }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __w[__shift / __chunksize + __i]; })); } } // }}} // __extract_part(_SimdWrapper<_Tp, _Np>) {{{ template <int _Index, int _Total, int _Combine, typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr _SimdWrapper<_Tp, _Np / _Total * _Combine> __extract_part(const _SimdWrapper<_Tp, _Np> __x) { if constexpr (_Index % 2 == 0 && _Total % 2 == 0 && _Combine % 2 == 0) return __extract_part<_Index / 2, _Total / 2, _Combine / 2>(__x); else { constexpr size_t __values_per_part = _Np / _Total; constexpr size_t __values_to_skip = _Index * __values_per_part; constexpr size_t __return_size = __values_per_part * _Combine; using _R = __vector_type_t<_Tp, __return_size>; static_assert((_Index + _Combine) * __values_per_part * sizeof(_Tp) <= sizeof(__x), "out of bounds __extract_part"); // the following assertion would ensure no "padding" to be read // static_assert(_Total >= _Index + _Combine, "_Total must be greater // than _Index"); // static_assert(__return_size * _Total == _Np, "_Np must be divisible // by _Total"); if (__x._M_is_constprop()) return __generate_from_n_evaluations<__return_size, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__values_to_skip + __i]; }); if constexpr (_Index == 0 && _Total == 1) return __x; else if constexpr (_Index == 0) return __intrin_bitcast<_R>(__as_vector(__x)); #if _GLIBCXX_SIMD_X86INTRIN // {{{ else if constexpr (sizeof(__x) == 32 && __return_size * sizeof(_Tp) <= 16) { constexpr size_t __bytes_to_skip = __values_to_skip * sizeof(_Tp); if constexpr (__bytes_to_skip == 16) return __vector_bitcast<_Tp, __return_size>( __hi128(__as_vector(__x))); else return __vector_bitcast<_Tp, __return_size>( _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)), __lo128(__vector_bitcast<_LLong>(__x)), __bytes_to_skip)); } #endif // _GLIBCXX_SIMD_X86INTRIN }}} else if constexpr (_Index > 0 && (__values_to_skip % __return_size != 0 || sizeof(_R) >= 8) && (__values_to_skip + __return_size) * sizeof(_Tp) <= 64 && sizeof(__x) >= 16) return __intrin_bitcast<_R>( __shift_elements_right<__values_to_skip * sizeof(_Tp)>( __as_vector(__x))); else { _R __r = {}; __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + sizeof(_Tp) * __values_to_skip, __return_size * sizeof(_Tp)); return __r; } } } // }}} // __extract_part(_SimdWrapper<bool, _Np>) {{{ template <int _Index, int _Total, int _Combine = 1, size_t _Np> _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _Np / _Total * _Combine> __extract_part(const _SimdWrapper<bool, _Np> __x) { static_assert(_Combine == 1, "_Combine != 1 not implemented"); static_assert(__have_avx512f && _Total >= 2 && _Index + _Combine <= _Total && _Index >= 0); return __x._M_data >> (_Index * _Np / _Total); } // }}} // __vector_convert {{{ // implementation requires an index sequence template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, _From __l, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, _From __l, _From __m, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])..., static_cast<_Tp>(__m[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, _From __l, _From __m, _From __n, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])..., static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, _From __l, _From __m, _From __n, _From __o, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])..., static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])..., static_cast<_Tp>(__o[_I])...}; } template <typename _To, typename _From, size_t... _I> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e, _From __f, _From __g, _From __h, _From __i, _From __j, _From __k, _From __l, _From __m, _From __n, _From __o, _From __p, index_sequence<_I...>) { using _Tp = typename _VectorTraits<_To>::value_type; return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])..., static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])..., static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])..., static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])..., static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])..., static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])..., static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])..., static_cast<_Tp>(__o[_I])..., static_cast<_Tp>(__p[_I])...}; } // Defer actual conversion to the overload that takes an index sequence. Note // that this function adds zeros or drops values off the end if you don't ensure // matching width. template <typename _To, typename... _From, size_t _FromSize> _GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_SimdWrapper<_From, _FromSize>... __xs) { #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048 using _From0 = __first_of_pack_t<_From...>; using _FW = _SimdWrapper<_From0, _FromSize>; if (!_FW::_S_is_partial && !(... && __xs._M_is_constprop())) { if constexpr ((sizeof...(_From) & (sizeof...(_From) - 1)) == 0) // power-of-two number of arguments return __convert_x86<_To>(__as_vector(__xs)...); else // append zeros and recurse until the above branch is taken return __vector_convert<_To>(__xs..., _FW{}); } else #endif return __vector_convert<_To>( __as_vector(__xs)..., make_index_sequence<(sizeof...(__xs) == 1 ? std::min( _VectorTraits<_To>::_S_full_size, int(_FromSize)) : _FromSize)>()); } // }}} // __convert function{{{ template <typename _To, typename _From, typename... _More> _GLIBCXX_SIMD_INTRINSIC constexpr auto __convert(_From __v0, _More... __vs) { static_assert((true && ... && is_same_v<_From, _More>) ); if constexpr (__is_vectorizable_v<_From>) { using _V = typename _VectorTraits<_To>::type; using _Tp = typename _VectorTraits<_To>::value_type; return _V{static_cast<_Tp>(__v0), static_cast<_Tp>(__vs)...}; } else if constexpr (__is_vector_type_v<_From>) return __convert<_To>(__as_wrapper(__v0), __as_wrapper(__vs)...); else // _SimdWrapper arguments { constexpr size_t __input_size = _From::_S_size * (1 + sizeof...(_More)); if constexpr (__is_vectorizable_v<_To>) return __convert<__vector_type_t<_To, __input_size>>(__v0, __vs...); else if constexpr (!__is_vector_type_v<_To>) return _To(__convert<typename _To::_BuiltinType>(__v0, __vs...)); else { static_assert( sizeof...(_More) == 0 || _VectorTraits<_To>::_S_full_size >= __input_size, "__convert(...) requires the input to fit into the output"); return __vector_convert<_To>(__v0, __vs...); } } } // }}} // __convert_all{{{ // Converts __v into array<_To, N>, where N is _NParts if non-zero or // otherwise deduced from _To such that N * #elements(_To) <= #elements(__v). // Note: this function may return less than all converted elements template <typename _To, size_t _NParts = 0, // allows to convert fewer or more (only last // _To, to be partially filled) than all size_t _Offset = 0, // where to start, # of elements (not Bytes or // Parts) typename _From, typename _FromVT = _VectorTraits<_From>> _GLIBCXX_SIMD_INTRINSIC auto __convert_all(_From __v) { if constexpr (is_arithmetic_v<_To> && _NParts != 1) { static_assert(_Offset < _FromVT::_S_full_size); constexpr auto _Np = _NParts == 0 ? _FromVT::_S_partial_width - _Offset : _NParts; return __generate_from_n_evaluations<_Np, array<_To, _Np>>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return static_cast<_To>(__v[__i + _Offset]); }); } else { static_assert(__is_vector_type_v<_To>); using _ToVT = _VectorTraits<_To>; if constexpr (__is_vector_type_v<_From>) return __convert_all<_To, _NParts>(__as_wrapper(__v)); else if constexpr (_NParts == 1) { static_assert(_Offset % _ToVT::_S_full_size == 0); return array<_To, 1>{__vector_convert<_To>( __extract_part<_Offset / _ToVT::_S_full_size, __div_roundup(_FromVT::_S_partial_width, _ToVT::_S_full_size)>(__v))}; } #if _GLIBCXX_SIMD_X86INTRIN // {{{ else if constexpr (!__have_sse4_1 && _Offset == 0 && is_integral_v<typename _FromVT::value_type> && sizeof(typename _FromVT::value_type) < sizeof(typename _ToVT::value_type) && !(sizeof(typename _FromVT::value_type) == 4 && is_same_v<typename _ToVT::value_type, double>)) { using _ToT = typename _ToVT::value_type; using _FromT = typename _FromVT::value_type; constexpr size_t _Np = _NParts != 0 ? _NParts : (_FromVT::_S_partial_width / _ToVT::_S_full_size); using _R = array<_To, _Np>; // __adjust modifies its input to have _Np (use _SizeConstant) // entries so that no unnecessary intermediate conversions are // requested and, more importantly, no intermediate conversions are // missing [[maybe_unused]] auto __adjust = [](auto __n, auto __vv) -> _SimdWrapper<_FromT, decltype(__n)::value> { return __vector_bitcast<_FromT, decltype(__n)::value>(__vv); }; [[maybe_unused]] const auto __vi = __to_intrin(__v); auto&& __make_array = [](auto __x0, [[maybe_unused]] auto __x1) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if constexpr (_Np == 1) return _R{__intrin_bitcast<_To>(__x0)}; else return _R{__intrin_bitcast<_To>(__x0), __intrin_bitcast<_To>(__x1)}; }; if constexpr (_Np == 0) return _R{}; else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) == 2) { static_assert(is_integral_v<_FromT>); static_assert(is_integral_v<_ToT>); if constexpr (is_unsigned_v<_FromT>) return __make_array(_mm_unpacklo_epi8(__vi, __m128i()), _mm_unpackhi_epi8(__vi, __m128i())); else return __make_array( _mm_srai_epi16(_mm_unpacklo_epi8(__vi, __vi), 8), _mm_srai_epi16(_mm_unpackhi_epi8(__vi, __vi), 8)); } else if constexpr (sizeof(_FromT) == 2 && sizeof(_ToT) == 4) { static_assert(is_integral_v<_FromT>); if constexpr (is_floating_point_v<_ToT>) { const auto __ints = __convert_all<__vector_type16_t<int>, _Np>( __adjust(_SizeConstant<_Np * 4>(), __v)); return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __vector_convert<_To>(__as_wrapper(__ints[__i])); }); } else if constexpr (is_unsigned_v<_FromT>) return __make_array(_mm_unpacklo_epi16(__vi, __m128i()), _mm_unpackhi_epi16(__vi, __m128i())); else return __make_array( _mm_srai_epi32(_mm_unpacklo_epi16(__vi, __vi), 16), _mm_srai_epi32(_mm_unpackhi_epi16(__vi, __vi), 16)); } else if constexpr (sizeof(_FromT) == 4 && sizeof(_ToT) == 8 && is_integral_v<_FromT> && is_integral_v<_ToT>) { if constexpr (is_unsigned_v<_FromT>) return __make_array(_mm_unpacklo_epi32(__vi, __m128i()), _mm_unpackhi_epi32(__vi, __m128i())); else return __make_array( _mm_unpacklo_epi32(__vi, _mm_srai_epi32(__vi, 31)), _mm_unpackhi_epi32(__vi, _mm_srai_epi32(__vi, 31))); } else if constexpr (sizeof(_FromT) == 4 && sizeof(_ToT) == 8 && is_integral_v<_FromT> && is_integral_v<_ToT>) { if constexpr (is_unsigned_v<_FromT>) return __make_array(_mm_unpacklo_epi32(__vi, __m128i()), _mm_unpackhi_epi32(__vi, __m128i())); else return __make_array( _mm_unpacklo_epi32(__vi, _mm_srai_epi32(__vi, 31)), _mm_unpackhi_epi32(__vi, _mm_srai_epi32(__vi, 31))); } else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) >= 4 && is_signed_v<_FromT>) { const __m128i __vv[2] = {_mm_unpacklo_epi8(__vi, __vi), _mm_unpackhi_epi8(__vi, __vi)}; const __vector_type_t<int, 4> __vvvv[4] = { __vector_bitcast<int>(_mm_unpacklo_epi16(__vv[0], __vv[0])), __vector_bitcast<int>(_mm_unpackhi_epi16(__vv[0], __vv[0])), __vector_bitcast<int>(_mm_unpacklo_epi16(__vv[1], __vv[1])), __vector_bitcast<int>(_mm_unpackhi_epi16(__vv[1], __vv[1]))}; if constexpr (sizeof(_ToT) == 4) return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __vector_convert<_To>( _SimdWrapper<int, 4>(__vvvv[__i] >> 24)); }); else if constexpr (is_integral_v<_ToT>) return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { const auto __signbits = __to_intrin(__vvvv[__i / 2] >> 31); const auto __sx32 = __to_intrin(__vvvv[__i / 2] >> 24); return __vector_bitcast<_ToT>( __i % 2 == 0 ? _mm_unpacklo_epi32(__sx32, __signbits) : _mm_unpackhi_epi32(__sx32, __signbits)); }); else return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { const _SimdWrapper<int, 4> __int4 = __vvvv[__i / 2] >> 24; return __vector_convert<_To>( __i % 2 == 0 ? __int4 : _SimdWrapper<int, 4>( _mm_unpackhi_epi64(__to_intrin(__int4), __to_intrin(__int4)))); }); } else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) == 4) { const auto __shorts = __convert_all<__vector_type16_t< conditional_t<is_signed_v<_FromT>, short, unsigned short>>>( __adjust(_SizeConstant<(_Np + 1) / 2 * 8>(), __v)); return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __convert_all<_To>(__shorts[__i / 2])[__i % 2]; }); } else if constexpr (sizeof(_FromT) == 2 && sizeof(_ToT) == 8 && is_signed_v<_FromT> && is_integral_v<_ToT>) { const __m128i __vv[2] = {_mm_unpacklo_epi16(__vi, __vi), _mm_unpackhi_epi16(__vi, __vi)}; const __vector_type16_t<int> __vvvv[4] = {__vector_bitcast<int>( _mm_unpacklo_epi32(_mm_srai_epi32(__vv[0], 16), _mm_srai_epi32(__vv[0], 31))), __vector_bitcast<int>( _mm_unpackhi_epi32(_mm_srai_epi32(__vv[0], 16), _mm_srai_epi32(__vv[0], 31))), __vector_bitcast<int>( _mm_unpacklo_epi32(_mm_srai_epi32(__vv[1], 16), _mm_srai_epi32(__vv[1], 31))), __vector_bitcast<int>( _mm_unpackhi_epi32(_mm_srai_epi32(__vv[1], 16), _mm_srai_epi32(__vv[1], 31)))}; return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __vector_bitcast<_ToT>(__vvvv[__i]); }); } else if constexpr (sizeof(_FromT) <= 2 && sizeof(_ToT) == 8) { const auto __ints = __convert_all<__vector_type16_t<conditional_t< is_signed_v<_FromT> || is_floating_point_v<_ToT>, int, unsigned int>>>( __adjust(_SizeConstant<(_Np + 1) / 2 * 4>(), __v)); return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __convert_all<_To>(__ints[__i / 2])[__i % 2]; }); } else __assert_unreachable<_To>(); } #endif // _GLIBCXX_SIMD_X86INTRIN }}} else if constexpr ((_FromVT::_S_partial_width - _Offset) > _ToVT::_S_full_size) { /* static_assert( (_FromVT::_S_partial_width & (_FromVT::_S_partial_width - 1)) == 0, "__convert_all only supports power-of-2 number of elements. Otherwise " "the return type cannot be array<_To, N>."); */ constexpr size_t _NTotal = (_FromVT::_S_partial_width - _Offset) / _ToVT::_S_full_size; constexpr size_t _Np = _NParts == 0 ? _NTotal : _NParts; static_assert( _Np <= _NTotal || (_Np == _NTotal + 1 && (_FromVT::_S_partial_width - _Offset) % _ToVT::_S_full_size > 0)); using _R = array<_To, _Np>; if constexpr (_Np == 1) return _R{__vector_convert<_To>( __extract_part<_Offset, _FromVT::_S_partial_width, _ToVT::_S_full_size>(__v))}; else return __generate_from_n_evaluations<_Np, _R>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { auto __part = __extract_part<__i * _ToVT::_S_full_size + _Offset, _FromVT::_S_partial_width, _ToVT::_S_full_size>(__v); return __vector_convert<_To>(__part); }); } else if constexpr (_Offset == 0) return array<_To, 1>{__vector_convert<_To>(__v)}; else return array<_To, 1>{__vector_convert<_To>( __extract_part<_Offset, _FromVT::_S_partial_width, _FromVT::_S_partial_width - _Offset>(__v))}; } } // }}} // _GnuTraits {{{ template <typename _Tp, typename _Mp, typename _Abi, size_t _Np> struct _GnuTraits { using _IsValid = true_type; using _SimdImpl = typename _Abi::_SimdImpl; using _MaskImpl = typename _Abi::_MaskImpl; // simd and simd_mask member types {{{ using _SimdMember = _SimdWrapper<_Tp, _Np>; using _MaskMember = _SimdWrapper<_Mp, _Np>; static constexpr size_t _S_simd_align = alignof(_SimdMember); static constexpr size_t _S_mask_align = alignof(_MaskMember); // }}} // size metadata {{{ static constexpr size_t _S_full_size = _SimdMember::_S_full_size; static constexpr bool _S_is_partial = _SimdMember::_S_is_partial; // }}} // _SimdBase / base class for simd, providing extra conversions {{{ struct _SimdBase2 { explicit operator __intrinsic_type_t<_Tp, _Np>() const { return __to_intrin(static_cast<const simd<_Tp, _Abi>*>(this)->_M_data); } explicit operator __vector_type_t<_Tp, _Np>() const { return __data(*static_cast<const simd<_Tp, _Abi>*>(this)); } }; struct _SimdBase1 { explicit operator __intrinsic_type_t<_Tp, _Np>() const { return __data(*static_cast<const simd<_Tp, _Abi>*>(this)); } }; using _SimdBase = conditional_t< is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value, _SimdBase1, _SimdBase2>; // }}} // _MaskBase {{{ struct _MaskBase2 { explicit operator __intrinsic_type_t<_Tp, _Np>() const { return static_cast<const simd_mask<_Tp, _Abi>*>(this) ->_M_data.__intrin(); } explicit operator __vector_type_t<_Tp, _Np>() const { return static_cast<const simd_mask<_Tp, _Abi>*>(this)->_M_data._M_data; } }; struct _MaskBase1 { explicit operator __intrinsic_type_t<_Tp, _Np>() const { return __data(*static_cast<const simd_mask<_Tp, _Abi>*>(this)); } }; using _MaskBase = conditional_t< is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value, _MaskBase1, _MaskBase2>; // }}} // _MaskCastType {{{ // parameter type of one explicit simd_mask constructor class _MaskCastType { using _Up = __intrinsic_type_t<_Tp, _Np>; _Up _M_data; public: _MaskCastType(_Up __x) : _M_data(__x) {} operator _MaskMember() const { return _M_data; } }; // }}} // _SimdCastType {{{ // parameter type of one explicit simd constructor class _SimdCastType1 { using _Ap = __intrinsic_type_t<_Tp, _Np>; _SimdMember _M_data; public: constexpr _SimdCastType1(_Ap __a) : _M_data(__vector_bitcast<_Tp>(__a)) {} constexpr operator _SimdMember() const { return _M_data; } }; class _SimdCastType2 { using _Ap = __intrinsic_type_t<_Tp, _Np>; using _Bp = __vector_type_t<_Tp, _Np>; _SimdMember _M_data; public: constexpr _SimdCastType2(_Ap __a) : _M_data(__vector_bitcast<_Tp>(__a)) {} constexpr _SimdCastType2(_Bp __b) : _M_data(__b) {} constexpr operator _SimdMember() const { return _M_data; } }; using _SimdCastType = conditional_t< is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value, _SimdCastType1, _SimdCastType2>; //}}} }; // }}} struct _CommonImplX86; struct _CommonImplNeon; struct _CommonImplBuiltin; template <typename _Abi> struct _SimdImplBuiltin; template <typename _Abi> struct _MaskImplBuiltin; template <typename _Abi> struct _SimdImplX86; template <typename _Abi> struct _MaskImplX86; template <typename _Abi> struct _SimdImplNeon; template <typename _Abi> struct _MaskImplNeon; template <typename _Abi> struct _SimdImplPpc; template <typename _Abi> struct _MaskImplPpc; // simd_abi::_VecBuiltin {{{ template <int _UsedBytes> struct simd_abi::_VecBuiltin { template <typename _Tp> static constexpr size_t _S_size = _UsedBytes / sizeof(_Tp); // validity traits {{{ struct _IsValidAbiTag : __bool_constant<(_UsedBytes > 1)> {}; template <typename _Tp> struct _IsValidSizeFor : __bool_constant<(_UsedBytes / sizeof(_Tp) > 1 && _UsedBytes % sizeof(_Tp) == 0 && _UsedBytes <= __vectorized_sizeof<_Tp>() && (!__have_avx512f || _UsedBytes <= 32))> {}; template <typename _Tp> struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>, _IsValidSizeFor<_Tp>> {}; template <typename _Tp> static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value; // }}} // _SimdImpl/_MaskImpl {{{ #if _GLIBCXX_SIMD_X86INTRIN using _CommonImpl = _CommonImplX86; using _SimdImpl = _SimdImplX86<_VecBuiltin<_UsedBytes>>; using _MaskImpl = _MaskImplX86<_VecBuiltin<_UsedBytes>>; #elif _GLIBCXX_SIMD_HAVE_NEON using _CommonImpl = _CommonImplNeon; using _SimdImpl = _SimdImplNeon<_VecBuiltin<_UsedBytes>>; using _MaskImpl = _MaskImplNeon<_VecBuiltin<_UsedBytes>>; #else using _CommonImpl = _CommonImplBuiltin; #ifdef __ALTIVEC__ using _SimdImpl = _SimdImplPpc<_VecBuiltin<_UsedBytes>>; using _MaskImpl = _MaskImplPpc<_VecBuiltin<_UsedBytes>>; #else using _SimdImpl = _SimdImplBuiltin<_VecBuiltin<_UsedBytes>>; using _MaskImpl = _MaskImplBuiltin<_VecBuiltin<_UsedBytes>>; #endif #endif // }}} // __traits {{{ template <typename _Tp> using _MaskValueType = __int_for_sizeof_t<_Tp>; template <typename _Tp> using __traits = conditional_t<_S_is_valid_v<_Tp>, _GnuTraits<_Tp, _MaskValueType<_Tp>, _VecBuiltin<_UsedBytes>, _S_size<_Tp>>, _InvalidTraits>; //}}} // size metadata {{{ template <typename _Tp> static constexpr size_t _S_full_size = __traits<_Tp>::_S_full_size; template <typename _Tp> static constexpr bool _S_is_partial = __traits<_Tp>::_S_is_partial; // }}} // implicit masks {{{ template <typename _Tp> using _MaskMember = _SimdWrapper<_MaskValueType<_Tp>, _S_size<_Tp>>; template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_implicit_mask() { using _UV = typename _MaskMember<_Tp>::_BuiltinType; if constexpr (!_MaskMember<_Tp>::_S_is_partial) return ~_UV(); else { constexpr auto __size = _S_size<_Tp>; _GLIBCXX_SIMD_USE_CONSTEXPR auto __r = __generate_vector<_UV>([](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __i < __size ? -1 : 0; }); return __r; } } template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr __intrinsic_type_t<_Tp, _S_size<_Tp>> _S_implicit_mask_intrin() { return __to_intrin(__vector_bitcast<_Tp>(_S_implicit_mask<_Tp>()._M_data)); } template <typename _TW, typename _TVT = _VectorTraits<_TW>> _GLIBCXX_SIMD_INTRINSIC static constexpr _TW _S_masked(_TW __x) { using _Tp = typename _TVT::value_type; if constexpr (!_MaskMember<_Tp>::_S_is_partial) return __x; else return __and(__as_vector(__x), __vector_bitcast<_Tp>(_S_implicit_mask<_Tp>())); } template <typename _TW, typename _TVT = _VectorTraits<_TW>> _GLIBCXX_SIMD_INTRINSIC static constexpr auto __make_padding_nonzero(_TW __x) { using _Tp = typename _TVT::value_type; if constexpr (!_S_is_partial<_Tp>) return __x; else { _GLIBCXX_SIMD_USE_CONSTEXPR auto __implicit_mask = __vector_bitcast<_Tp>(_S_implicit_mask<_Tp>()); if constexpr (is_integral_v<_Tp>) return __or(__x, ~__implicit_mask); else { _GLIBCXX_SIMD_USE_CONSTEXPR auto __one = __andnot(__implicit_mask, __vector_broadcast<_S_full_size<_Tp>>(_Tp(1))); // it's not enough to return `x | 1_in_padding` because the // padding in x might be inf or nan (independent of // __FINITE_MATH_ONLY__, because it's about padding bits) return __or(__and(__x, __implicit_mask), __one); } } } // }}} }; // }}} // simd_abi::_VecBltnBtmsk {{{ template <int _UsedBytes> struct simd_abi::_VecBltnBtmsk { template <typename _Tp> static constexpr size_t _S_size = _UsedBytes / sizeof(_Tp); // validity traits {{{ struct _IsValidAbiTag : __bool_constant<(_UsedBytes > 1)> {}; template <typename _Tp> struct _IsValidSizeFor : __bool_constant<(_UsedBytes / sizeof(_Tp) > 1 && _UsedBytes % sizeof(_Tp) == 0 && _UsedBytes <= 64 && (_UsedBytes > 32 || __have_avx512vl))> {}; // Bitmasks require at least AVX512F. If sizeof(_Tp) < 4 the AVX512BW is also // required. template <typename _Tp> struct _IsValid : conjunction< _IsValidAbiTag, __bool_constant<__have_avx512f>, __bool_constant<__have_avx512bw || (sizeof(_Tp) >= 4)>, __bool_constant<(__vectorized_sizeof<_Tp>() > sizeof(_Tp))>, _IsValidSizeFor<_Tp>> {}; template <typename _Tp> static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value; // }}} // simd/_MaskImpl {{{ #if _GLIBCXX_SIMD_X86INTRIN using _CommonImpl = _CommonImplX86; using _SimdImpl = _SimdImplX86<_VecBltnBtmsk<_UsedBytes>>; using _MaskImpl = _MaskImplX86<_VecBltnBtmsk<_UsedBytes>>; #else template <int> struct _MissingImpl; using _CommonImpl = _MissingImpl<_UsedBytes>; using _SimdImpl = _MissingImpl<_UsedBytes>; using _MaskImpl = _MissingImpl<_UsedBytes>; #endif // }}} // __traits {{{ template <typename _Tp> using _MaskMember = _SimdWrapper<bool, _S_size<_Tp>>; template <typename _Tp> using __traits = conditional_t< _S_is_valid_v<_Tp>, _GnuTraits<_Tp, bool, _VecBltnBtmsk<_UsedBytes>, _S_size<_Tp>>, _InvalidTraits>; //}}} // size metadata {{{ template <typename _Tp> static constexpr size_t _S_full_size = __traits<_Tp>::_S_full_size; template <typename _Tp> static constexpr bool _S_is_partial = __traits<_Tp>::_S_is_partial; // }}} // implicit mask {{{ private: template <typename _Tp> using _ImplicitMask = _SimdWrapper<bool, _S_size<_Tp>>; public: template <size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr __bool_storage_member_type_t<_Np> __implicit_mask_n() { using _Tp = __bool_storage_member_type_t<_Np>; return _Np < sizeof(_Tp) * __CHAR_BIT__ ? _Tp((1ULL << _Np) - 1) : ~_Tp(); } template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _ImplicitMask<_Tp> _S_implicit_mask() { return __implicit_mask_n<_S_size<_Tp>>(); } template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr __bool_storage_member_type_t<_S_size<_Tp>> _S_implicit_mask_intrin() { return __implicit_mask_n<_S_size<_Tp>>(); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_masked(_SimdWrapper<_Tp, _Np> __x) { if constexpr (is_same_v<_Tp, bool>) if constexpr (_Np < 8 || (_Np & (_Np - 1)) != 0) return _MaskImpl::_S_bit_and( __x, _SimdWrapper<_Tp, _Np>( __bool_storage_member_type_t<_Np>((1ULL << _Np) - 1))); else return __x; else return _S_masked(__x._M_data); } template <typename _TV> _GLIBCXX_SIMD_INTRINSIC static constexpr _TV _S_masked(_TV __x) { using _Tp = typename _VectorTraits<_TV>::value_type; static_assert( !__is_bitmask_v<_TV>, "_VecBltnBtmsk::_S_masked cannot work on bitmasks, since it doesn't " "know the number of elements. Use _SimdWrapper<bool, N> instead."); if constexpr (_S_is_partial<_Tp>) { constexpr size_t _Np = _S_size<_Tp>; return __make_dependent_t<_TV, _CommonImpl>::_S_blend( _S_implicit_mask<_Tp>(), _SimdWrapper<_Tp, _Np>(), _SimdWrapper<_Tp, _Np>(__x)); } else return __x; } template <typename _TV, typename _TVT = _VectorTraits<_TV>> _GLIBCXX_SIMD_INTRINSIC static constexpr auto __make_padding_nonzero(_TV __x) { using _Tp = typename _TVT::value_type; if constexpr (!_S_is_partial<_Tp>) return __x; else { constexpr size_t _Np = _S_size<_Tp>; if constexpr (is_integral_v<typename _TVT::value_type>) return __x | __generate_vector<_Tp, _S_full_size<_Tp>>( [](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Tp { if (__i < _Np) return 0; else return 1; }); else return __make_dependent_t<_TV, _CommonImpl>::_S_blend( _S_implicit_mask<_Tp>(), _SimdWrapper<_Tp, _Np>( __vector_broadcast<_S_full_size<_Tp>>(_Tp(1))), _SimdWrapper<_Tp, _Np>(__x)) ._M_data; } } // }}} }; //}}} // _CommonImplBuiltin {{{ struct _CommonImplBuiltin { // _S_converts_via_decomposition{{{ // This lists all cases where a __vector_convert needs to fall back to // conversion of individual scalars (i.e. decompose the input vector into // scalars, convert, compose output vector). In those cases, _S_masked_load & // _S_masked_store prefer to use the _S_bit_iteration implementation. template <typename _From, typename _To, size_t _ToSize> static inline constexpr bool __converts_via_decomposition_v = sizeof(_From) != sizeof(_To); // }}} // _S_load{{{ template <typename _Tp, size_t _Np, size_t _Bytes = _Np * sizeof(_Tp)> _GLIBCXX_SIMD_INTRINSIC static __vector_type_t<_Tp, _Np> _S_load(const void* __p) { static_assert(_Np > 1); static_assert(_Bytes % sizeof(_Tp) == 0); using _Rp = __vector_type_t<_Tp, _Np>; if constexpr (sizeof(_Rp) == _Bytes) { _Rp __r; __builtin_memcpy(&__r, __p, _Bytes); return __r; } else { #ifdef _GLIBCXX_SIMD_WORKAROUND_PR90424 using _Up = conditional_t< is_integral_v<_Tp>, conditional_t<_Bytes % 4 == 0, conditional_t<_Bytes % 8 == 0, long long, int>, conditional_t<_Bytes % 2 == 0, short, signed char>>, conditional_t<(_Bytes < 8 || _Np % 2 == 1 || _Np == 2), _Tp, double>>; using _V = __vector_type_t<_Up, _Np * sizeof(_Tp) / sizeof(_Up)>; if constexpr (sizeof(_V) != sizeof(_Rp)) { // on i386 with 4 < _Bytes <= 8 _Rp __r{}; __builtin_memcpy(&__r, __p, _Bytes); return __r; } else #else // _GLIBCXX_SIMD_WORKAROUND_PR90424 using _V = _Rp; #endif // _GLIBCXX_SIMD_WORKAROUND_PR90424 { _V __r{}; static_assert(_Bytes <= sizeof(_V)); __builtin_memcpy(&__r, __p, _Bytes); return reinterpret_cast<_Rp>(__r); } } } // }}} // _S_store {{{ template <size_t _Bytes> _GLIBCXX_SIMD_INTRINSIC static void _S_memcpy(char* __dst, const char* __src) { if constexpr (_Bytes > 0) { constexpr size_t _Ns = std::__bit_floor(_Bytes); __builtin_memcpy(__dst, __src, _Ns); _S_memcpy<_Bytes - _Ns>(__dst + _Ns, __src + _Ns); } } template <size_t _ReqBytes = 0, typename _TV> _GLIBCXX_SIMD_INTRINSIC static void _S_store(_TV __x, void* __addr) { constexpr size_t _Bytes = _ReqBytes == 0 ? sizeof(__x) : _ReqBytes; static_assert(sizeof(__x) >= _Bytes); #if !defined __clang__ && _GLIBCXX_SIMD_WORKAROUND_PR90424 if constexpr (__is_vector_type_v<_TV>) _S_memcpy<_Bytes>(reinterpret_cast<char*>(__addr), reinterpret_cast<const char*>(&__x)); else #endif // _GLIBCXX_SIMD_WORKAROUND_PR90424 __builtin_memcpy(__addr, &__x, _Bytes); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static void _S_store(_SimdWrapper<_Tp, _Np> __x, void* __addr) { _S_store<_Np * sizeof(_Tp)>(__x._M_data, __addr); } // }}} // _S_store_bool_array(_BitMask) {{{ template <size_t _Np, bool _Sanitized> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_store_bool_array(_BitMask<_Np, _Sanitized> __x, bool* __mem) { if constexpr (_Np == 1) __mem[0] = __x[0]; else if (__builtin_is_constant_evaluated()) { for (size_t __i = 0; __i < _Np; ++__i) __mem[__i] = __x[__i]; } else if constexpr (_Np == 2) { short __bool2 = (__x._M_to_bits() * 0x81) & 0x0101; _S_store<_Np>(__bool2, __mem); } else if constexpr (_Np == 3) { int __bool3 = (__x._M_to_bits() * 0x4081) & 0x010101; _S_store<_Np>(__bool3, __mem); } else { __execute_n_times<__div_roundup(_Np, 4)>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { constexpr int __offset = __i * 4; constexpr int __remaining = _Np - __offset; if constexpr (__remaining > 4 && __remaining <= 7) { const _ULLong __bool7 = (__x.template _M_extract<__offset>()._M_to_bits() * 0x40810204081ULL) & 0x0101010101010101ULL; _S_store<__remaining>(__bool7, __mem + __offset); } else if constexpr (__remaining >= 4) { int __bits = __x.template _M_extract<__offset>()._M_to_bits(); if constexpr (__remaining > 7) __bits &= 0xf; const int __bool4 = (__bits * 0x204081) & 0x01010101; _S_store<4>(__bool4, __mem + __offset); } }); } } // }}} // _S_blend{{{ template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr auto _S_blend(_SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k, _SimdWrapper<_Tp, _Np> __at0, _SimdWrapper<_Tp, _Np> __at1) { return __k._M_data ? __at1._M_data : __at0._M_data; } // }}} }; // }}} // _SimdImplBuiltin {{{1 template <typename _Abi> struct _SimdImplBuiltin { // member types {{{2 template <typename _Tp> static constexpr size_t _S_max_store_size = 16; using abi_type = _Abi; template <typename _Tp> using _TypeTag = _Tp*; template <typename _Tp> using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember; template <typename _Tp> using _MaskMember = typename _Abi::template _MaskMember<_Tp>; template <typename _Tp> static constexpr size_t _S_size = _Abi::template _S_size<_Tp>; template <typename _Tp> static constexpr size_t _S_full_size = _Abi::template _S_full_size<_Tp>; using _CommonImpl = typename _Abi::_CommonImpl; using _SuperImpl = typename _Abi::_SimdImpl; using _MaskImpl = typename _Abi::_MaskImpl; // _M_make_simd(_SimdWrapper/__intrinsic_type_t) {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr simd<_Tp, _Abi> _M_make_simd(_SimdWrapper<_Tp, _Np> __x) { return {__private_init, __x}; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr simd<_Tp, _Abi> _M_make_simd(__intrinsic_type_t<_Tp, _Np> __x) { return {__private_init, __vector_bitcast<_Tp>(__x)}; } // _S_broadcast {{{2 template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> _S_broadcast(_Tp __x) noexcept { return __vector_broadcast<_S_full_size<_Tp>>(__x); } // _S_generator {{{2 template <typename _Fp, typename _Tp> inline static constexpr _SimdMember<_Tp> _S_generator(_Fp&& __gen, _TypeTag<_Tp>) { return __generate_vector<_Tp, _S_full_size<_Tp>>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if constexpr (__i < _S_size<_Tp>) return __gen(__i); else return 0; }); } // _S_load {{{2 template <typename _Tp, typename _Up> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> _S_load(const _Up* __mem, _TypeTag<_Tp>) noexcept { constexpr size_t _Np = _S_size<_Tp>; constexpr size_t __max_load_size = (sizeof(_Up) >= 4 && __have_avx512f) || __have_avx512bw ? 64 : (is_floating_point_v<_Up> && __have_avx) || __have_avx2 ? 32 : 16; constexpr size_t __bytes_to_load = sizeof(_Up) * _Np; if (__builtin_is_constant_evaluated()) return __generate_vector<_Tp, _S_full_size<_Tp>>( [&](auto __i) constexpr { return static_cast<_Tp>(__i < _Np ? __mem[__i] : 0); }); else if constexpr (sizeof(_Up) > 8 or __vectorized_sizeof<_Up>() <= sizeof(_Up)) return __generate_vector<_Tp, _SimdMember<_Tp>::_S_full_size>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return static_cast<_Tp>(__i < _Np ? __mem[__i] : 0); }); else if constexpr (is_same_v<_Up, _Tp>) return _CommonImpl::template _S_load<_Tp, _S_full_size<_Tp>, _Np * sizeof(_Tp)>(__mem); else if constexpr (__bytes_to_load <= __max_load_size) return __convert<_SimdMember<_Tp>>( _CommonImpl::template _S_load<_Up, _Np>(__mem)); else if constexpr (__bytes_to_load % __max_load_size == 0) { constexpr size_t __n_loads = __bytes_to_load / __max_load_size; constexpr size_t __elements_per_load = _Np / __n_loads; return __call_with_n_evaluations<__n_loads>( [](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __convert<_SimdMember<_Tp>>(__uncvted...); }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return _CommonImpl::template _S_load<_Up, __elements_per_load>( __mem + __i * __elements_per_load); }); } else if constexpr (__bytes_to_load % (__max_load_size / 2) == 0 && __max_load_size > 16) { // e.g. int[] -> <char, 12> with AVX2 constexpr size_t __n_loads = __bytes_to_load / (__max_load_size / 2); constexpr size_t __elements_per_load = _Np / __n_loads; return __call_with_n_evaluations<__n_loads>( [](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __convert<_SimdMember<_Tp>>(__uncvted...); }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return _CommonImpl::template _S_load<_Up, __elements_per_load>( __mem + __i * __elements_per_load); }); } else // e.g. int[] -> <char, 9> return __call_with_subscripts( __mem, make_index_sequence<_Np>(), [](auto... __args) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __vector_type_t<_Tp, _S_full_size<_Tp>>{static_cast<_Tp>(__args)...}; }); } // _S_masked_load {{{2 template <typename _Tp, size_t _Np, typename _Up> static constexpr inline _SimdWrapper<_Tp, _Np> _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k, const _Up* __mem) noexcept { _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k), [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __merge._M_set(__i, static_cast<_Tp>(__mem[__i])); }); return __merge; } // _S_store {{{2 template <typename _Tp, typename _Up> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_store(_SimdMember<_Tp> __v, _Up* __mem, _TypeTag<_Tp>) noexcept { // TODO: converting int -> "smaller int" can be optimized with AVX512 constexpr size_t _Np = _S_size<_Tp>; constexpr size_t __max_store_size = _SuperImpl::template _S_max_store_size<_Up>; if (__builtin_is_constant_evaluated()) { for (size_t __i = 0; __i < _Np; ++__i) __mem[__i] = __v[__i]; } else if constexpr (sizeof(_Up) > 8 or __vectorized_sizeof<_Up>() <= sizeof(_Up)) __execute_n_times<_Np>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; }); else if constexpr (is_same_v<_Up, _Tp>) _CommonImpl::_S_store(__v, __mem); else if constexpr (sizeof(_Up) * _Np <= __max_store_size) _CommonImpl::_S_store(_SimdWrapper<_Up, _Np>(__convert<_Up>(__v)), __mem); else { constexpr size_t __vsize = __max_store_size / sizeof(_Up); // round up to convert the last partial vector as well: constexpr size_t __stores = __div_roundup(_Np, __vsize); constexpr size_t __full_stores = _Np / __vsize; using _V = __vector_type_t<_Up, __vsize>; const array<_V, __stores> __converted = __convert_all<_V, __stores>(__v); __execute_n_times<__full_stores>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _CommonImpl::_S_store(__converted[__i], __mem + __i * __vsize); }); if constexpr (__full_stores < __stores) _CommonImpl::template _S_store<(_Np - __full_stores * __vsize) * sizeof(_Up)>( __converted[__full_stores], __mem + __full_stores * __vsize); } } // _S_masked_store_nocvt {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem, _MaskMember<_Tp> __k) { _BitOps::_S_bit_iteration( _MaskImpl::_S_to_bits(__k), [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; }); } // _S_masked_store {{{2 template <typename _TW, typename _TVT = _VectorTraits<_TW>, typename _Tp = typename _TVT::value_type, typename _Up> static constexpr inline void _S_masked_store(const _TW __v, _Up* __mem, const _MaskMember<_Tp> __k) noexcept { constexpr size_t _TV_size = _S_size<_Tp>; [[maybe_unused]] const auto __vi = __to_intrin(__v); constexpr size_t __max_store_size = _SuperImpl::template _S_max_store_size<_Up>; if constexpr ( is_same_v< _Tp, _Up> || (is_integral_v<_Tp> && is_integral_v<_Up> && sizeof(_Tp) == sizeof(_Up))) { // bitwise or no conversion, reinterpret: const _MaskMember<_Up> __kk = [&]() _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if constexpr (__is_bitmask_v<decltype(__k)>) return _MaskMember<_Up>(__k._M_data); else return __wrapper_bitcast<__int_for_sizeof_t<_Up>>(__k); }(); _SuperImpl::_S_masked_store_nocvt(__wrapper_bitcast<_Up>(__v), __mem, __kk); } else if constexpr (__vectorized_sizeof<_Up>() > sizeof(_Up) && !_CommonImpl:: template __converts_via_decomposition_v< _Tp, _Up, __max_store_size>) { // conversion via decomposition is better handled via the // bit_iteration // fallback below constexpr size_t _UW_size = std::min(_TV_size, __max_store_size / sizeof(_Up)); static_assert(_UW_size <= _TV_size); using _UW = _SimdWrapper<_Up, _UW_size>; using _UV = __vector_type_t<_Up, _UW_size>; using _UAbi = simd_abi::deduce_t<_Up, _UW_size>; if constexpr (_UW_size == _TV_size) // one convert+store { const _UW __converted = __convert<_UW>(__v); _UAbi::_SimdImpl::_S_masked_store_nocvt( __converted, __mem, _UAbi::_MaskImpl::template _S_convert< __int_for_sizeof_t<_Up>>(__k)); } else { static_assert(_UW_size * sizeof(_Up) == __max_store_size); constexpr size_t _NFullStores = _TV_size / _UW_size; constexpr size_t _NAllStores = __div_roundup(_TV_size, _UW_size); constexpr size_t _NParts = _S_full_size<_Tp> / _UW_size; const array<_UV, _NAllStores> __converted = __convert_all<_UV, _NAllStores>(__v); __execute_n_times<_NFullStores>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _UAbi::_SimdImpl::_S_masked_store_nocvt( _UW(__converted[__i]), __mem + __i * _UW_size, _UAbi::_MaskImpl::template _S_convert< __int_for_sizeof_t<_Up>>( __extract_part<__i, _NParts>(__k.__as_full_vector()))); }); if constexpr (_NAllStores > _NFullStores) // one partial at the end _UAbi::_SimdImpl::_S_masked_store_nocvt( _UW(__converted[_NFullStores]), __mem + _NFullStores * _UW_size, _UAbi::_MaskImpl::template _S_convert< __int_for_sizeof_t<_Up>>( __extract_part<_NFullStores, _NParts>( __k.__as_full_vector()))); } } else _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k), [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = static_cast<_Up>(__v[__i]); }); } // _S_complement {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_complement(_SimdWrapper<_Tp, _Np> __x) noexcept { return ~__x._M_data; } // _S_unary_minus {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_unary_minus(_SimdWrapper<_Tp, _Np> __x) noexcept { // GCC doesn't use the psign instructions, but pxor & psub seem to be // just as good a choice as pcmpeqd & psign. So meh. return -__x._M_data; } // arithmetic operators {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_plus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data + __y._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_minus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data - __y._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_multiplies(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data * __y._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_divides(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { // Note that division by 0 is always UB, so we must ensure we avoid the // case for partial registers if constexpr (!_Abi::template _S_is_partial<_Tp>) return __x._M_data / __y._M_data; else return __x._M_data / _Abi::__make_padding_nonzero(__y._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_modulus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { if constexpr (!_Abi::template _S_is_partial<_Tp>) return __x._M_data % __y._M_data; else return __as_vector(__x) % _Abi::__make_padding_nonzero(__as_vector(__y)); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_and(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __and(__x, __y); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_or(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __or(__x, __y); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_xor(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __xor(__x, __y); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_bit_shift_left(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data << __y._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_bit_shift_right(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data >> __y._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_shift_left(_SimdWrapper<_Tp, _Np> __x, int __y) { return __x._M_data << __y; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_shift_right(_SimdWrapper<_Tp, _Np> __x, int __y) { return __x._M_data >> __y; } // compares {{{2 // _S_equal_to {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data == __y._M_data; } // _S_not_equal_to {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_not_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data != __y._M_data; } // _S_less {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_less(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data < __y._M_data; } // _S_less_equal {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_less_equal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __x._M_data <= __y._M_data; } // _S_negate {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_negate(_SimdWrapper<_Tp, _Np> __x) noexcept { return !__x._M_data; } // _S_min, _S_max, _S_minmax {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_min(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b) { return __a._M_data < __b._M_data ? __a._M_data : __b._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_max(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b) { return __a._M_data > __b._M_data ? __a._M_data : __b._M_data; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr pair<_SimdWrapper<_Tp, _Np>, _SimdWrapper<_Tp, _Np>> _S_minmax(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b) { return {__a._M_data < __b._M_data ? __a._M_data : __b._M_data, __a._M_data < __b._M_data ? __b._M_data : __a._M_data}; } // reductions {{{2 template <size_t _Np, size_t... _Is, size_t... _Zeros, typename _Tp, typename _BinaryOperation> _GLIBCXX_SIMD_INTRINSIC static constexpr _Tp _S_reduce_partial(index_sequence<_Is...>, index_sequence<_Zeros...>, simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op) { using _V = __vector_type_t<_Tp, _Np / 2>; static_assert(sizeof(_V) <= sizeof(__x)); // _S_full_size is the size of the smallest native SIMD register that // can store _Np/2 elements: using _FullSimd = __deduced_simd<_Tp, _VectorTraits<_V>::_S_full_size>; using _HalfSimd = __deduced_simd<_Tp, _Np / 2>; const auto __xx = __as_vector(__x); return _HalfSimd::abi_type::_SimdImpl::_S_reduce( static_cast<_HalfSimd>(__as_vector(__binary_op( static_cast<_FullSimd>(__intrin_bitcast<_V>(__xx)), static_cast<_FullSimd>(__intrin_bitcast<_V>( __vector_permute<(_Np / 2 + _Is)..., (int(_Zeros * 0) - 1)...>( __xx)))))), __binary_op); } template <typename _Tp, typename _BinaryOperation> _GLIBCXX_SIMD_INTRINSIC static constexpr _Tp _S_reduce(simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op) { constexpr size_t _Np = simd_size_v<_Tp, _Abi>; if constexpr (_Np == 1) return __x[0]; else if constexpr (_Np == 2) return __binary_op(simd<_Tp, simd_abi::scalar>(__x[0]), simd<_Tp, simd_abi::scalar>(__x[1]))[0]; else if (__builtin_is_constant_evaluated()) { simd<_Tp, simd_abi::scalar> __acc = __x[0]; for (size_t __i = 1; __i < _Np; ++__i) __acc = __binary_op(__acc, simd<_Tp, simd_abi::scalar>(__x[__i])); return __acc[0]; } else if constexpr (_Abi::template _S_is_partial<_Tp>) //{{{ { [[maybe_unused]] constexpr auto __full_size = _Abi::template _S_full_size<_Tp>; if constexpr (_Np == 3) return __binary_op( __binary_op(simd<_Tp, simd_abi::scalar>(__x[0]), simd<_Tp, simd_abi::scalar>(__x[1])), simd<_Tp, simd_abi::scalar>(__x[2]))[0]; else if constexpr (is_same_v<__remove_cvref_t<_BinaryOperation>, plus<>>) { using _Ap = simd_abi::deduce_t<_Tp, __full_size>; return _Ap::_SimdImpl::_S_reduce( simd<_Tp, _Ap>(__private_init, _Abi::_S_masked(__as_vector(__x))), __binary_op); } else if constexpr (is_same_v<__remove_cvref_t<_BinaryOperation>, multiplies<>>) { using _Ap = simd_abi::deduce_t<_Tp, __full_size>; using _TW = _SimdWrapper<_Tp, __full_size>; _GLIBCXX_SIMD_USE_CONSTEXPR auto __implicit_mask_full = _Abi::template _S_implicit_mask<_Tp>().__as_full_vector(); _GLIBCXX_SIMD_USE_CONSTEXPR _TW __one = __vector_broadcast<__full_size>(_Tp(1)); const _TW __x_full = __data(__x).__as_full_vector(); const _TW __x_padded_with_ones = _Ap::_CommonImpl::_S_blend(__implicit_mask_full, __one, __x_full); return _Ap::_SimdImpl::_S_reduce( simd<_Tp, _Ap>(__private_init, __x_padded_with_ones), __binary_op); } else if constexpr (_Np & 1) { using _Ap = simd_abi::deduce_t<_Tp, _Np - 1>; return __binary_op( simd<_Tp, simd_abi::scalar>(_Ap::_SimdImpl::_S_reduce( simd<_Tp, _Ap>( __intrin_bitcast<__vector_type_t<_Tp, _Np - 1>>( __as_vector(__x))), __binary_op)), simd<_Tp, simd_abi::scalar>(__x[_Np - 1]))[0]; } else return _S_reduce_partial<_Np>( make_index_sequence<_Np / 2>(), make_index_sequence<__full_size - _Np / 2>(), __x, __binary_op); } //}}} else if constexpr (sizeof(__x) == 16) //{{{ { if constexpr (_Np == 16) { const auto __y = __data(__x); __x = __binary_op( _M_make_simd<_Tp, _Np>( __vector_permute<0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7>(__y)), _M_make_simd<_Tp, _Np>( __vector_permute<8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15>(__y))); } if constexpr (_Np >= 8) { const auto __y = __vector_bitcast<short>(__data(__x)); __x = __binary_op( _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>( __vector_permute<0, 0, 1, 1, 2, 2, 3, 3>(__y))), _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>( __vector_permute<4, 4, 5, 5, 6, 6, 7, 7>(__y)))); } if constexpr (_Np >= 4) { using _Up = conditional_t<is_floating_point_v<_Tp>, float, int>; const auto __y = __vector_bitcast<_Up>(__data(__x)); __x = __binary_op(__x, _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>( __vector_permute<3, 2, 1, 0>(__y)))); } using _Up = conditional_t<is_floating_point_v<_Tp>, double, _LLong>; const auto __y = __vector_bitcast<_Up>(__data(__x)); __x = __binary_op(__x, _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>( __vector_permute<1, 1>(__y)))); return __x[0]; } //}}} else { static_assert(sizeof(__x) > __min_vector_size<_Tp>); static_assert((_Np & (_Np - 1)) == 0); // _Np must be a power of 2 using _Ap = simd_abi::deduce_t<_Tp, _Np / 2>; using _V = simd<_Tp, _Ap>; return _Ap::_SimdImpl::_S_reduce( __binary_op(_V(__private_init, __extract<0, 2>(__as_vector(__x))), _V(__private_init, __extract<1, 2>(__as_vector(__x)))), static_cast<_BinaryOperation&&>(__binary_op)); } } // math {{{2 // frexp, modf and copysign implemented in simd_math.h #define _GLIBCXX_SIMD_MATH_FALLBACK(__name) \ template <typename _Tp, typename... _More> \ static _Tp \ _S_##__name(const _Tp& __x, const _More&... __more) \ { \ return __generate_vector<_Tp>( \ [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ return __name(__x[__i], __more[__i]...); \ }); \ } #define _GLIBCXX_SIMD_MATH_FALLBACK_MASKRET(__name) \ template <typename _Tp, typename... _More> \ static typename _Tp::mask_type \ _S_##__name(const _Tp& __x, const _More&... __more) \ { \ return __generate_vector<_Tp>( \ [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ return __name(__x[__i], __more[__i]...); \ }); \ } #define _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(_RetTp, __name) \ template <typename _Tp, typename... _More> \ static auto \ _S_##__name(const _Tp& __x, const _More&... __more) \ { \ return __fixed_size_storage_t<_RetTp, \ _VectorTraits<_Tp>::_S_partial_width>:: \ _S_generate([&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ return __meta._S_generator( \ [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ return __name(__x[__meta._S_offset + __i], \ __more[__meta._S_offset + __i]...); \ }, \ static_cast<_RetTp*>(nullptr)); \ }); \ } _GLIBCXX_SIMD_MATH_FALLBACK(acos) _GLIBCXX_SIMD_MATH_FALLBACK(asin) _GLIBCXX_SIMD_MATH_FALLBACK(atan) _GLIBCXX_SIMD_MATH_FALLBACK(atan2) _GLIBCXX_SIMD_MATH_FALLBACK(cos) _GLIBCXX_SIMD_MATH_FALLBACK(sin) _GLIBCXX_SIMD_MATH_FALLBACK(tan) _GLIBCXX_SIMD_MATH_FALLBACK(acosh) _GLIBCXX_SIMD_MATH_FALLBACK(asinh) _GLIBCXX_SIMD_MATH_FALLBACK(atanh) _GLIBCXX_SIMD_MATH_FALLBACK(cosh) _GLIBCXX_SIMD_MATH_FALLBACK(sinh) _GLIBCXX_SIMD_MATH_FALLBACK(tanh) _GLIBCXX_SIMD_MATH_FALLBACK(exp) _GLIBCXX_SIMD_MATH_FALLBACK(exp2) _GLIBCXX_SIMD_MATH_FALLBACK(expm1) _GLIBCXX_SIMD_MATH_FALLBACK(ldexp) _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(int, ilogb) _GLIBCXX_SIMD_MATH_FALLBACK(log) _GLIBCXX_SIMD_MATH_FALLBACK(log10) _GLIBCXX_SIMD_MATH_FALLBACK(log1p) _GLIBCXX_SIMD_MATH_FALLBACK(log2) _GLIBCXX_SIMD_MATH_FALLBACK(logb) // modf implemented in simd_math.h _GLIBCXX_SIMD_MATH_FALLBACK(scalbn) _GLIBCXX_SIMD_MATH_FALLBACK(scalbln) _GLIBCXX_SIMD_MATH_FALLBACK(cbrt) _GLIBCXX_SIMD_MATH_FALLBACK(fabs) _GLIBCXX_SIMD_MATH_FALLBACK(pow) _GLIBCXX_SIMD_MATH_FALLBACK(sqrt) _GLIBCXX_SIMD_MATH_FALLBACK(erf) _GLIBCXX_SIMD_MATH_FALLBACK(erfc) _GLIBCXX_SIMD_MATH_FALLBACK(lgamma) _GLIBCXX_SIMD_MATH_FALLBACK(tgamma) _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long, lrint) _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long long, llrint) _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long, lround) _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long long, llround) _GLIBCXX_SIMD_MATH_FALLBACK(fmod) _GLIBCXX_SIMD_MATH_FALLBACK(remainder) template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> static _Tp _S_remquo(const _Tp __x, const _Tp __y, __fixed_size_storage_t<int, _TVT::_S_partial_width>* __z) { return __generate_vector<_Tp>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { int __tmp; auto __r = remquo(__x[__i], __y[__i], &__tmp); __z->_M_set(__i, __tmp); return __r; }); } // copysign in simd_math.h _GLIBCXX_SIMD_MATH_FALLBACK(nextafter) _GLIBCXX_SIMD_MATH_FALLBACK(fdim) _GLIBCXX_SIMD_MATH_FALLBACK(fmax) _GLIBCXX_SIMD_MATH_FALLBACK(fmin) _GLIBCXX_SIMD_MATH_FALLBACK(fma) template <typename _Tp, size_t _Np> static constexpr _MaskMember<_Tp> _S_isgreater(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept { using _Ip = __int_for_sizeof_t<_Tp>; const auto __xn = __vector_bitcast<_Ip>(__x); const auto __yn = __vector_bitcast<_Ip>(__y); const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn; const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn; return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data, __xp > __yp); } template <typename _Tp, size_t _Np> static constexpr _MaskMember<_Tp> _S_isgreaterequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept { using _Ip = __int_for_sizeof_t<_Tp>; const auto __xn = __vector_bitcast<_Ip>(__x); const auto __yn = __vector_bitcast<_Ip>(__y); const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn; const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn; return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data, __xp >= __yp); } template <typename _Tp, size_t _Np> static constexpr _MaskMember<_Tp> _S_isless(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept { using _Ip = __int_for_sizeof_t<_Tp>; const auto __xn = __vector_bitcast<_Ip>(__x); const auto __yn = __vector_bitcast<_Ip>(__y); const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn; const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn; return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data, __xp < __yp); } template <typename _Tp, size_t _Np> static constexpr _MaskMember<_Tp> _S_islessequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept { using _Ip = __int_for_sizeof_t<_Tp>; const auto __xn = __vector_bitcast<_Ip>(__x); const auto __yn = __vector_bitcast<_Ip>(__y); const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn; const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn; return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data, __xp <= __yp); } template <typename _Tp, size_t _Np> static constexpr _MaskMember<_Tp> _S_islessgreater(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept { return __andnot(_SuperImpl::_S_isunordered(__x, __y), _SuperImpl::_S_not_equal_to(__x, __y)); } #undef _GLIBCXX_SIMD_MATH_FALLBACK #undef _GLIBCXX_SIMD_MATH_FALLBACK_MASKRET #undef _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET // _S_abs {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_abs(_SimdWrapper<_Tp, _Np> __x) noexcept { // if (__builtin_is_constant_evaluated()) // { // return __x._M_data < 0 ? -__x._M_data : __x._M_data; // } if constexpr (is_floating_point_v<_Tp>) // `v < 0 ? -v : v` cannot compile to the efficient implementation of // masking the signbit off because it must consider v == -0 // ~(-0.) & v would be easy, but breaks with fno-signed-zeros return __and(_S_absmask<__vector_type_t<_Tp, _Np>>, __x._M_data); else return __x._M_data < 0 ? -__x._M_data : __x._M_data; } // }}}3 // _S_plus_minus {{{ // Returns __x + __y - __y without -fassociative-math optimizing to __x. // - _TV must be __vector_type_t<floating-point type, N>. // - _UV must be _TV or floating-point type. template <typename _TV, typename _UV> _GLIBCXX_SIMD_INTRINSIC static constexpr _TV _S_plus_minus(_TV __x, _UV __y) noexcept { #if defined __i386__ && !defined __SSE_MATH__ if constexpr (sizeof(__x) == 8) { // operations on __x would use the FPU static_assert(is_same_v<_TV, __vector_type_t<float, 2>>); const auto __x4 = __vector_bitcast<float, 4>(__x); if constexpr (is_same_v<_TV, _UV>) return __vector_bitcast<float, 2>( _S_plus_minus(__x4, __vector_bitcast<float, 4>(__y))); else return __vector_bitcast<float, 2>(_S_plus_minus(__x4, __y)); } #endif #if !defined __clang__ && __GCC_IEC_559 == 0 if (__builtin_is_constant_evaluated() || (__builtin_constant_p(__x) && __builtin_constant_p(__y))) return (__x + __y) - __y; else return [&] { __x += __y; if constexpr(__have_sse) { if constexpr (sizeof(__x) >= 16) asm("" : "+x"(__x)); else if constexpr (is_same_v<__vector_type_t<float, 2>, _TV>) asm("" : "+x"(__x[0]), "+x"(__x[1])); else __assert_unreachable<_TV>(); } else if constexpr(__have_neon) asm("" : "+w"(__x)); else if constexpr (__have_power_vmx) { if constexpr (is_same_v<__vector_type_t<float, 2>, _TV>) asm("" : "+fgr"(__x[0]), "+fgr"(__x[1])); else asm("" : "+v"(__x)); } else asm("" : "+g"(__x)); return __x - __y; }(); #else return (__x + __y) - __y; #endif } // }}} // _S_nearbyint {{{3 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> _GLIBCXX_SIMD_INTRINSIC static _Tp _S_nearbyint(_Tp __x_) noexcept { using value_type = typename _TVT::value_type; using _V = typename _TVT::type; const _V __x = __x_; const _V __absx = __and(__x, _S_absmask<_V>); static_assert(__CHAR_BIT__ * sizeof(1ull) >= __digits_v<value_type>); _GLIBCXX_SIMD_USE_CONSTEXPR _V __shifter_abs = _V() + (1ull << (__digits_v<value_type> - 1)); const _V __shifter = __or(__and(_S_signmask<_V>, __x), __shifter_abs); const _V __shifted = _S_plus_minus(__x, __shifter); return __absx < __shifter_abs ? __shifted : __x; } // _S_rint {{{3 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> _GLIBCXX_SIMD_INTRINSIC static _Tp _S_rint(_Tp __x) noexcept { return _SuperImpl::_S_nearbyint(__x); } // _S_trunc {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_trunc(_SimdWrapper<_Tp, _Np> __x) { using _V = __vector_type_t<_Tp, _Np>; const _V __absx = __and(__x._M_data, _S_absmask<_V>); static_assert(__CHAR_BIT__ * sizeof(1ull) >= __digits_v<_Tp>); constexpr _Tp __shifter = 1ull << (__digits_v<_Tp> - 1); _V __truncated = _S_plus_minus(__absx, __shifter); __truncated -= __truncated > __absx ? _V() + 1 : _V(); return __absx < __shifter ? __or(__xor(__absx, __x._M_data), __truncated) : __x._M_data; } // _S_round {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_round(_SimdWrapper<_Tp, _Np> __x) { const auto __abs_x = _SuperImpl::_S_abs(__x); const auto __t_abs = _SuperImpl::_S_trunc(__abs_x)._M_data; const auto __r_abs // round(abs(x)) = = __t_abs + (__abs_x._M_data - __t_abs >= _Tp(.5) ? _Tp(1) : 0); return __or(__xor(__abs_x._M_data, __x._M_data), __r_abs); } // _S_floor {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_floor(_SimdWrapper<_Tp, _Np> __x) { const auto __y = _SuperImpl::_S_trunc(__x)._M_data; const auto __negative_input = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_Np, _Tp>(0)); const auto __mask = __andnot(__vector_bitcast<_Tp>(__y == __x._M_data), __negative_input); return __or(__andnot(__mask, __y), __and(__mask, __y - __vector_broadcast<_Np, _Tp>(1))); } // _S_ceil {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np> _S_ceil(_SimdWrapper<_Tp, _Np> __x) { const auto __y = _SuperImpl::_S_trunc(__x)._M_data; const auto __negative_input = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_Np, _Tp>(0)); const auto __inv_mask = __or(__vector_bitcast<_Tp>(__y == __x._M_data), __negative_input); return __or(__and(__inv_mask, __y), __andnot(__inv_mask, __y + __vector_broadcast<_Np, _Tp>(1))); } // _S_isnan {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_isnan([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x) { #if __FINITE_MATH_ONLY__ return {}; // false #elif !defined __SUPPORT_SNAN__ return ~(__x._M_data == __x._M_data); #elif defined __STDC_IEC_559__ using _Ip = __int_for_sizeof_t<_Tp>; const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x)); const auto __infn = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__infinity_v<_Tp>)); return __infn < __absn; #else #error "Not implemented: how to support SNaN but non-IEC559 floating-point?" #endif } // _S_isfinite {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_isfinite([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x) { #if __FINITE_MATH_ONLY__ using _UV = typename _MaskMember<_Tp>::_BuiltinType; _GLIBCXX_SIMD_USE_CONSTEXPR _UV __alltrue = ~_UV(); return __alltrue; #else // if all exponent bits are set, __x is either inf or NaN using _Ip = __int_for_sizeof_t<_Tp>; const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x)); const auto __maxn = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__finite_max_v<_Tp>)); return __absn <= __maxn; #endif } // _S_isunordered {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_isunordered(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) { return __or(_S_isnan(__x), _S_isnan(__y)); } // _S_signbit {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_signbit(_SimdWrapper<_Tp, _Np> __x) { using _Ip = __int_for_sizeof_t<_Tp>; return __vector_bitcast<_Ip>(__x) < 0; // Arithmetic right shift (SRA) would also work (instead of compare), but // 64-bit SRA isn't available on x86 before AVX512. And in general, // compares are more likely to be efficient than SRA. } // _S_isinf {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_isinf([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x) { #if __FINITE_MATH_ONLY__ return {}; // false #else return _SuperImpl::template _S_equal_to<_Tp, _Np>(_SuperImpl::_S_abs(__x), __vector_broadcast<_Np>( __infinity_v<_Tp>)); // alternative: // compare to inf using the corresponding integer type /* return __vector_bitcast<_Tp>(__vector_bitcast<__int_for_sizeof_t<_Tp>>( _S_abs(__x)._M_data) == __vector_bitcast<__int_for_sizeof_t<_Tp>>(__vector_broadcast<_Np>( __infinity_v<_Tp>))); */ #endif } // _S_isnormal {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_isnormal(_SimdWrapper<_Tp, _Np> __x) { using _Ip = __int_for_sizeof_t<_Tp>; const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x)); const auto __minn = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__norm_min_v<_Tp>)); #if __FINITE_MATH_ONLY__ return __absn >= __minn; #else const auto __maxn = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__finite_max_v<_Tp>)); return __minn <= __absn && __absn <= __maxn; #endif } // _S_fpclassify {{{3 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static __fixed_size_storage_t<int, _Np> _S_fpclassify(_SimdWrapper<_Tp, _Np> __x) { using _I = __int_for_sizeof_t<_Tp>; const auto __xn = __vector_bitcast<_I>(__to_intrin(_SuperImpl::_S_abs(__x))); constexpr size_t _NI = sizeof(__xn) / sizeof(_I); _GLIBCXX_SIMD_USE_CONSTEXPR auto __minn = __vector_bitcast<_I>(__vector_broadcast<_NI>(__norm_min_v<_Tp>)); _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_normal = __vector_broadcast<_NI, _I>(FP_NORMAL); #if !__FINITE_MATH_ONLY__ _GLIBCXX_SIMD_USE_CONSTEXPR auto __infn = __vector_bitcast<_I>(__vector_broadcast<_NI>(__infinity_v<_Tp>)); _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_nan = __vector_broadcast<_NI, _I>(FP_NAN); _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_infinite = __vector_broadcast<_NI, _I>(FP_INFINITE); #endif #ifndef __FAST_MATH__ _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_subnormal = __vector_broadcast<_NI, _I>(FP_SUBNORMAL); #endif _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_zero = __vector_broadcast<_NI, _I>(FP_ZERO); __vector_type_t<_I, _NI> __tmp = __xn < __minn #ifdef __FAST_MATH__ ? __fp_zero #else ? (__xn == 0 ? __fp_zero : __fp_subnormal) #endif #if __FINITE_MATH_ONLY__ : __fp_normal; #else : (__xn < __infn ? __fp_normal : (__xn == __infn ? __fp_infinite : __fp_nan)); #endif if constexpr (sizeof(_I) == sizeof(int)) { using _FixedInt = __fixed_size_storage_t<int, _Np>; const auto __as_int = __vector_bitcast<int, _Np>(__tmp); if constexpr (_FixedInt::_S_tuple_size == 1) return {__as_int}; else if constexpr (_FixedInt::_S_tuple_size == 2 && is_same_v< typename _FixedInt::_SecondType::_FirstAbi, simd_abi::scalar>) return {__extract<0, 2>(__as_int), __as_int[_Np - 1]}; else if constexpr (_FixedInt::_S_tuple_size == 2) return {__extract<0, 2>(__as_int), __auto_bitcast(__extract<1, 2>(__as_int))}; else __assert_unreachable<_Tp>(); } else if constexpr (_Np == 2 && sizeof(_I) == 8 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 2) { const auto __aslong = __vector_bitcast<_LLong>(__tmp); return {int(__aslong[0]), {int(__aslong[1])}}; } #if _GLIBCXX_SIMD_X86INTRIN else if constexpr (sizeof(_Tp) == 8 && sizeof(__tmp) == 32 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 1) return {_mm_packs_epi32(__to_intrin(__lo128(__tmp)), __to_intrin(__hi128(__tmp)))}; else if constexpr (sizeof(_Tp) == 8 && sizeof(__tmp) == 64 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 1) return {_mm512_cvtepi64_epi32(__to_intrin(__tmp))}; #endif // _GLIBCXX_SIMD_X86INTRIN else if constexpr (__fixed_size_storage_t<int, _Np>::_S_tuple_size == 1) return {__call_with_subscripts<_Np>(__vector_bitcast<_LLong>(__tmp), [](auto... __l) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __make_wrapper<int>(__l...); })}; else __assert_unreachable<_Tp>(); } // _S_increment & _S_decrement{{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_increment(_SimdWrapper<_Tp, _Np>& __x) { __x = __x._M_data + 1; } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_decrement(_SimdWrapper<_Tp, _Np>& __x) { __x = __x._M_data - 1; } // smart_reference access {{{2 template <typename _Tp, size_t _Np, typename _Up> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_set(_SimdWrapper<_Tp, _Np>& __v, int __i, _Up&& __x) noexcept { __v._M_set(__i, static_cast<_Up&&>(__x)); } // _S_masked_assign{{{2 template <typename _Tp, typename _K, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_masked_assign(_SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs) { if (__k._M_is_constprop_none_of()) return; else if (__k._M_is_constprop_all_of()) __lhs = __rhs; else __lhs = _CommonImpl::_S_blend(__k, __lhs, __rhs); } template <typename _Tp, typename _K, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_masked_assign(_SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, __type_identity_t<_Tp> __rhs) { if (__k._M_is_constprop_none_of()) return; else if (__k._M_is_constprop_all_of()) __lhs = __vector_broadcast<_Np>(__rhs); else if (__builtin_constant_p(__rhs) && __rhs == 0) { if constexpr (!is_same_v<bool, _K>) // the __andnot optimization only makes sense if __k._M_data is a // vector register __lhs._M_data = __andnot(__vector_bitcast<_Tp>(__k), __lhs._M_data); else // for AVX512/__mmask, a _mm512_maskz_mov is best __lhs = _CommonImpl::_S_blend(__k, __lhs, _SimdWrapper<_Tp, _Np>()); } else __lhs = _CommonImpl::_S_blend(__k, __lhs, _SimdWrapper<_Tp, _Np>( __vector_broadcast<_Np>(__rhs))); } // _S_masked_cassign {{{2 template <typename _Op, typename _Tp, typename _K, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_masked_cassign(const _SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, const __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs, _Op __op) { if (__k._M_is_constprop_none_of()) return; else if (__k._M_is_constprop_all_of()) __lhs = __op(_SuperImpl{}, __lhs, __rhs); else __lhs = _CommonImpl::_S_blend(__k, __lhs, __op(_SuperImpl{}, __lhs, __rhs)); } template <typename _Op, typename _Tp, typename _K, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_masked_cassign(const _SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, const __type_identity_t<_Tp> __rhs, _Op __op) { _S_masked_cassign(__k, __lhs, __vector_broadcast<_Np>(__rhs), __op); } // _S_masked_unary {{{2 template <template <typename> class _Op, typename _Tp, typename _K, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_masked_unary(const _SimdWrapper<_K, _Np> __k, const _SimdWrapper<_Tp, _Np> __v) { if (__k._M_is_constprop_none_of()) return __v; auto __vv = _M_make_simd(__v); _Op<decltype(__vv)> __op; if (__k._M_is_constprop_all_of()) return __data(__op(__vv)); else if constexpr (is_same_v<_Op<void>, __increment<void>>) { static_assert(not std::is_same_v<_K, bool>); if constexpr (is_integral_v<_Tp>) // Take a shortcut knowing that __k is an integer vector with values -1 or 0. return __v._M_data - __vector_bitcast<_Tp>(__k._M_data); else if constexpr (not __have_avx2) return __v._M_data + __vector_bitcast<_Tp>(__k._M_data & __builtin_bit_cast( _K, _Tp(1))); // starting with AVX2 it is more efficient to blend after add } else if constexpr (is_same_v<_Op<void>, __decrement<void>>) { static_assert(not std::is_same_v<_K, bool>); if constexpr (is_integral_v<_Tp>) // Take a shortcut knowing that __k is an integer vector with values -1 or 0. return __v._M_data + __vector_bitcast<_Tp>(__k._M_data); else if constexpr (not __have_avx2) return __v._M_data - __vector_bitcast<_Tp>(__k._M_data & __builtin_bit_cast( _K, _Tp(1))); // starting with AVX2 it is more efficient to blend after sub } return _CommonImpl::_S_blend(__k, __v, __data(__op(__vv))); } //}}}2 }; // _MaskImplBuiltinMixin {{{1 struct _MaskImplBuiltinMixin { template <typename _Tp> using _TypeTag = _Tp*; // _S_to_maskvector {{{ template <typename _Up, size_t _ToN = 1> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN> _S_to_maskvector(bool __x) { static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>); return __x ? __vector_type_t<_Up, _ToN>{~_Up()} : __vector_type_t<_Up, _ToN>{}; } template <typename _Up, size_t _UpN = 0, size_t _Np, bool _Sanitized, size_t _ToN = _UpN == 0 ? _Np : _UpN> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN> _S_to_maskvector(_BitMask<_Np, _Sanitized> __x) { static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>); return __generate_vector<__vector_type_t<_Up, _ToN>>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if constexpr (__i < _Np) return __x[__i] ? ~_Up() : _Up(); else return _Up(); }); } template <typename _Up, size_t _UpN = 0, typename _Tp, size_t _Np, size_t _ToN = _UpN == 0 ? _Np : _UpN> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN> _S_to_maskvector(_SimdWrapper<_Tp, _Np> __x) { static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>); using _TW = _SimdWrapper<_Tp, _Np>; using _UW = _SimdWrapper<_Up, _ToN>; if constexpr (sizeof(_Up) == sizeof(_Tp) && sizeof(_TW) == sizeof(_UW)) return __wrapper_bitcast<_Up, _ToN>(__x); else if constexpr (is_same_v<_Tp, bool>) // bits -> vector return _S_to_maskvector<_Up, _ToN>(_BitMask<_Np>(__x._M_data)); else { // vector -> vector /* [[maybe_unused]] const auto __y = __vector_bitcast<_Up>(__x._M_data); if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4 && sizeof(__y) == 16) return __vector_permute<1, 3, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2 && sizeof(__y) == 16) return __vector_permute<1, 3, 5, 7, -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2 && sizeof(__y) == 16) return __vector_permute<3, 7, -1, -1, -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1 && sizeof(__y) == 16) return __vector_permute<1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1 && sizeof(__y) == 16) return __vector_permute<3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1 && sizeof(__y) == 16) return __vector_permute<7, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1>(__y); else */ { return __generate_vector<__vector_type_t<_Up, _ToN>>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if constexpr (__i < _Np) return _Up(__x[__i.value]); else return _Up(); }); } } } // }}} // _S_to_bits {{{ template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np> _S_to_bits(_SimdWrapper<_Tp, _Np> __x) { static_assert(!is_same_v<_Tp, bool>); static_assert(_Np <= __CHAR_BIT__ * sizeof(_ULLong)); using _Up = make_unsigned_t<__int_for_sizeof_t<_Tp>>; const auto __bools = __vector_bitcast<_Up>(__x) >> (sizeof(_Up) * __CHAR_BIT__ - 1); _ULLong __r = 0; __execute_n_times<_Np>( [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __r |= _ULLong(__bools[__i.value]) << __i; }); return __r; } // }}} }; // _MaskImplBuiltin {{{1 template <typename _Abi> struct _MaskImplBuiltin : _MaskImplBuiltinMixin { using _MaskImplBuiltinMixin::_S_to_bits; using _MaskImplBuiltinMixin::_S_to_maskvector; // member types {{{ template <typename _Tp> using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember; template <typename _Tp> using _MaskMember = typename _Abi::template _MaskMember<_Tp>; using _SuperImpl = typename _Abi::_MaskImpl; using _CommonImpl = typename _Abi::_CommonImpl; template <typename _Tp> static constexpr size_t _S_size = simd_size_v<_Tp, _Abi>; // }}} // _S_broadcast {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_broadcast(bool __x) { return __x ? _Abi::template _S_implicit_mask<_Tp>() : _MaskMember<_Tp>(); } // }}} // _S_load {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> _S_load(const bool* __mem) { using _I = __int_for_sizeof_t<_Tp>; if (not __builtin_is_constant_evaluated()) if constexpr (sizeof(_Tp) == sizeof(bool)) { const auto __bools = _CommonImpl::template _S_load<_I, _S_size<_Tp>>(__mem); // bool is {0, 1}, everything else is UB return __bools > 0; } return __generate_vector<_I, _S_size<_Tp>>( [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __mem[__i] ? ~_I() : _I(); }); } // }}} // _S_convert {{{ template <typename _Tp, size_t _Np, bool _Sanitized> _GLIBCXX_SIMD_INTRINSIC static constexpr auto _S_convert(_BitMask<_Np, _Sanitized> __x) { if constexpr (__is_builtin_bitmask_abi<_Abi>()) return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>(__x._M_to_bits()); else return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>, _S_size<_Tp>>( __x._M_sanitized()); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr auto _S_convert(_SimdWrapper<bool, _Np> __x) { if constexpr (__is_builtin_bitmask_abi<_Abi>()) return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>(__x._M_data); else return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>, _S_size<_Tp>>( _BitMask<_Np>(__x._M_data)._M_sanitized()); } template <typename _Tp, typename _Up, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr auto _S_convert(_SimdWrapper<_Up, _Np> __x) { if constexpr (__is_builtin_bitmask_abi<_Abi>()) return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>( _SuperImpl::_S_to_bits(__x)); else return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>, _S_size<_Tp>>(__x); } template <typename _Tp, typename _Up, typename _UAbi> _GLIBCXX_SIMD_INTRINSIC static constexpr auto _S_convert(simd_mask<_Up, _UAbi> __x) { if constexpr (__is_builtin_bitmask_abi<_Abi>()) { using _R = _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>; if constexpr (__is_builtin_bitmask_abi<_UAbi>()) // bits -> bits return _R(__data(__x)); else if constexpr (__is_scalar_abi<_UAbi>()) // bool -> bits return _R(__data(__x)); else if constexpr (__is_fixed_size_abi_v<_UAbi>) // bitset -> bits return _R(__data(__x)._M_to_bits()); else // vector -> bits return _R(_UAbi::_MaskImpl::_S_to_bits(__data(__x))._M_to_bits()); } else return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>, _S_size<_Tp>>( __data(__x)); } // }}} // _S_masked_load {{{2 template <typename _Tp, size_t _Np> static inline _SimdWrapper<_Tp, _Np> _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _SimdWrapper<_Tp, _Np> __mask, const bool* __mem) noexcept { // AVX(2) has 32/64 bit maskload, but nothing at 8 bit granularity auto __tmp = __wrapper_bitcast<__int_for_sizeof_t<_Tp>>(__merge); _BitOps::_S_bit_iteration(_SuperImpl::_S_to_bits(__mask), [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __tmp._M_set(__i, -__mem[__i]); }); __merge = __wrapper_bitcast<_Tp>(__tmp); return __merge; } // _S_store {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr void _S_store(_SimdWrapper<_Tp, _Np> __v, bool* __mem) noexcept { __execute_n_times<_Np>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; }); } // _S_masked_store {{{2 template <typename _Tp, size_t _Np> static inline void _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, bool* __mem, const _SimdWrapper<_Tp, _Np> __k) noexcept { _BitOps::_S_bit_iteration(_SuperImpl::_S_to_bits(__k), [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; }); } // _S_from_bitmask{{{2 template <size_t _Np, typename _Tp> _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> _S_from_bitmask(_SanitizedBitMask<_Np> __bits, _TypeTag<_Tp>) { return _SuperImpl::template _S_to_maskvector<_Tp, _S_size<_Tp>>(__bits); } // logical and bitwise operators {{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_logical_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y) { return __and(__x._M_data, __y._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_logical_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y) { return __or(__x._M_data, __y._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_not(const _SimdWrapper<_Tp, _Np>& __x) { if constexpr (_Abi::template _S_is_partial<_Tp>) return __andnot(__x, __wrapper_bitcast<_Tp>( _Abi::template _S_implicit_mask<_Tp>())); else return __not(__x._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y) { return __and(__x._M_data, __y._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y) { return __or(__x._M_data, __y._M_data); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np> _S_bit_xor(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y) { return __xor(__x._M_data, __y._M_data); } // smart_reference access {{{2 template <typename _Tp, size_t _Np> static constexpr void _S_set(_SimdWrapper<_Tp, _Np>& __k, int __i, bool __x) noexcept { if constexpr (is_same_v<_Tp, bool>) __k._M_set(__i, __x); else { static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>); if (__builtin_is_constant_evaluated()) { __k = __generate_from_n_evaluations<_Np, __vector_type_t<_Tp, _Np>>( [&](auto __j) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { if (__i == static_cast<int>(__j)) return _Tp(-__x); else return __k[+__j]; }); } else __k._M_data[__i] = -__x; } } // _S_masked_assign{{{2 template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static void _S_masked_assign(_SimdWrapper<_Tp, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs) { __lhs = _CommonImpl::_S_blend(__k, __lhs, __rhs); } template <typename _Tp, size_t _Np> _GLIBCXX_SIMD_INTRINSIC static void _S_masked_assign(_SimdWrapper<_Tp, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, bool __rhs) { if (__builtin_constant_p(__rhs)) { if (__rhs == false) __lhs = __andnot(__k, __lhs); else __lhs = __or(__k, __lhs); return; } __lhs = _CommonImpl::_S_blend(__k, __lhs, __data(simd_mask<_Tp, _Abi>(__rhs))); } //}}}2 // _S_all_of {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static bool _S_all_of(simd_mask<_Tp, _Abi> __k) { return __call_with_subscripts( __data(__k), make_index_sequence<_S_size<_Tp>>(), [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return (... && !(__ent == 0)); }); } // }}} // _S_any_of {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static bool _S_any_of(simd_mask<_Tp, _Abi> __k) { return __call_with_subscripts( __data(__k), make_index_sequence<_S_size<_Tp>>(), [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return (... || !(__ent == 0)); }); } // }}} // _S_none_of {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static bool _S_none_of(simd_mask<_Tp, _Abi> __k) { return __call_with_subscripts( __data(__k), make_index_sequence<_S_size<_Tp>>(), [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return (... && (__ent == 0)); }); } // }}} // _S_some_of {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static bool _S_some_of(simd_mask<_Tp, _Abi> __k) { const int __n_true = _SuperImpl::_S_popcount(__k); return __n_true > 0 && __n_true < int(_S_size<_Tp>); } // }}} // _S_popcount {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static int _S_popcount(simd_mask<_Tp, _Abi> __k) { using _I = __int_for_sizeof_t<_Tp>; if constexpr (is_default_constructible_v<simd<_I, _Abi>>) return -reduce( simd<_I, _Abi>(__private_init, __wrapper_bitcast<_I>(__data(__k)))); else return -reduce(__bit_cast<rebind_simd_t<_I, simd<_Tp, _Abi>>>( simd<_Tp, _Abi>(__private_init, __data(__k)))); } // }}} // _S_find_first_set {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static int _S_find_first_set(simd_mask<_Tp, _Abi> __k) { return std::__countr_zero(_SuperImpl::_S_to_bits(__data(__k))._M_to_bits()); } // }}} // _S_find_last_set {{{ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static int _S_find_last_set(simd_mask<_Tp, _Abi> __k) { return std::__bit_width(_SuperImpl::_S_to_bits(__data(__k))._M_to_bits()) - 1; } // }}} }; //}}}1 _GLIBCXX_SIMD_END_NAMESPACE #endif // __cplusplus >= 201703L #endif // _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_ // vim: foldmethod=marker foldmarker={{{,}}} sw=2 noet ts=8 sts=2 tw=80
💾 Save Changes
❌ Cancel