libstdc++
simd.h
1 // Definition of the public simd interfaces -*- C++ -*-
2 
3 // Copyright (C) 2020-2021 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
27 
28 #if __cplusplus >= 201703L
29 
30 #include "simd_detail.h"
31 #include "numeric_traits.h"
32 #include <bit>
33 #include <bitset>
34 #ifdef _GLIBCXX_DEBUG_UB
35 #include <cstdio> // for stderr
36 #endif
37 #include <cstring>
38 #include <functional>
39 #include <iosfwd>
40 #include <utility>
41 
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
45 #include <arm_neon.h>
46 #endif
47 
48 /** @ingroup ts_simd
49  * @{
50  */
51 /* There are several closely related types, with the following naming
52  * convention:
53  * _Tp: vectorizable (arithmetic) type (or any type)
54  * _TV: __vector_type_t<_Tp, _Np>
55  * _TW: _SimdWrapper<_Tp, _Np>
56  * _TI: __intrinsic_type_t<_Tp, _Np>
57  * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
58  * If one additional type is needed use _U instead of _T.
59  * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
60  *
61  * More naming conventions:
62  * _Ap or _Abi: An ABI tag from the simd_abi namespace
63  * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
64  * _IV, _IW as for _TV, _TW
65  * _Np: number of elements (not bytes)
66  * _Bytes: number of bytes
67  *
68  * Variable names:
69  * __k: mask object (vector- or bitmask)
70  */
71 _GLIBCXX_SIMD_BEGIN_NAMESPACE
72 
73 #if !_GLIBCXX_SIMD_X86INTRIN
74 using __m128 [[__gnu__::__vector_size__(16)]] = float;
75 using __m128d [[__gnu__::__vector_size__(16)]] = double;
76 using __m128i [[__gnu__::__vector_size__(16)]] = long long;
77 using __m256 [[__gnu__::__vector_size__(32)]] = float;
78 using __m256d [[__gnu__::__vector_size__(32)]] = double;
79 using __m256i [[__gnu__::__vector_size__(32)]] = long long;
80 using __m512 [[__gnu__::__vector_size__(64)]] = float;
81 using __m512d [[__gnu__::__vector_size__(64)]] = double;
82 using __m512i [[__gnu__::__vector_size__(64)]] = long long;
83 #endif
84 
85 namespace simd_abi {
86 // simd_abi forward declarations {{{
87 // implementation details:
88 struct _Scalar;
89 
90 template <int _Np>
91  struct _Fixed;
92 
93 // There are two major ABIs that appear on different architectures.
94 // Both have non-boolean values packed into an N Byte register
95 // -> #elements = N / sizeof(T)
96 // Masks differ:
97 // 1. Use value vector registers for masks (all 0 or all 1)
98 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding
99 // value vector
100 //
101 // Both can be partially used, masking off the rest when doing horizontal
102 // operations or operations that can trap (e.g. FP_INVALID or integer division
103 // by 0). This is encoded as the number of used bytes.
104 template <int _UsedBytes>
105  struct _VecBuiltin;
106 
107 template <int _UsedBytes>
108  struct _VecBltnBtmsk;
109 
110 template <typename _Tp, int _Np>
111  using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
112 
113 template <int _UsedBytes = 16>
114  using _Sse = _VecBuiltin<_UsedBytes>;
115 
116 template <int _UsedBytes = 32>
117  using _Avx = _VecBuiltin<_UsedBytes>;
118 
119 template <int _UsedBytes = 64>
120  using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
121 
122 template <int _UsedBytes = 16>
123  using _Neon = _VecBuiltin<_UsedBytes>;
124 
125 // implementation-defined:
126 using __sse = _Sse<>;
127 using __avx = _Avx<>;
128 using __avx512 = _Avx512<>;
129 using __neon = _Neon<>;
130 using __neon128 = _Neon<16>;
131 using __neon64 = _Neon<8>;
132 
133 // standard:
134 template <typename _Tp, size_t _Np, typename...>
135  struct deduce;
136 
137 template <int _Np>
138  using fixed_size = _Fixed<_Np>;
139 
140 using scalar = _Scalar;
141 
142 // }}}
143 } // namespace simd_abi
144 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{
145 template <typename _Tp>
146  struct is_simd;
147 
148 template <typename _Tp>
149  struct is_simd_mask;
150 
151 template <typename _Tp, typename _Abi>
152  class simd;
153 
154 template <typename _Tp, typename _Abi>
155  class simd_mask;
156 
157 template <typename _Tp, typename _Abi>
158  struct simd_size;
159 
160 // }}}
161 // load/store flags {{{
162 struct element_aligned_tag
163 {
164  template <typename _Tp, typename _Up = typename _Tp::value_type>
165  static constexpr size_t _S_alignment = alignof(_Up);
166 
167  template <typename _Tp, typename _Up>
168  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
169  _S_apply(_Up* __ptr)
170  { return __ptr; }
171 };
172 
173 struct vector_aligned_tag
174 {
175  template <typename _Tp, typename _Up = typename _Tp::value_type>
176  static constexpr size_t _S_alignment
177  = std::__bit_ceil(sizeof(_Up) * _Tp::size());
178 
179  template <typename _Tp, typename _Up>
180  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
181  _S_apply(_Up* __ptr)
182  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
183 };
184 
185 template <size_t _Np> struct overaligned_tag
186 {
187  template <typename _Tp, typename _Up = typename _Tp::value_type>
188  static constexpr size_t _S_alignment = _Np;
189 
190  template <typename _Tp, typename _Up>
191  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
192  _S_apply(_Up* __ptr)
193  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
194 };
195 
196 inline constexpr element_aligned_tag element_aligned = {};
197 
198 inline constexpr vector_aligned_tag vector_aligned = {};
199 
200 template <size_t _Np>
201  inline constexpr overaligned_tag<_Np> overaligned = {};
202 
203 // }}}
204 template <size_t _Xp>
205  using _SizeConstant = integral_constant<size_t, _Xp>;
206 
207 namespace __detail
208 {
209  struct _Minimum
210  {
211  template <typename _Tp>
212  _GLIBCXX_SIMD_INTRINSIC constexpr
213  _Tp
214  operator()(_Tp __a, _Tp __b) const
215  {
216  using std::min;
217  return min(__a, __b);
218  }
219  };
220 
221  struct _Maximum
222  {
223  template <typename _Tp>
224  _GLIBCXX_SIMD_INTRINSIC constexpr
225  _Tp
226  operator()(_Tp __a, _Tp __b) const
227  {
228  using std::max;
229  return max(__a, __b);
230  }
231  };
232 } // namespace __detail
233 
234 // unrolled/pack execution helpers
235 // __execute_n_times{{{
236 template <typename _Fp, size_t... _I>
237  _GLIBCXX_SIMD_INTRINSIC constexpr void
238  __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
239  { ((void)__f(_SizeConstant<_I>()), ...); }
240 
241 template <typename _Fp>
242  _GLIBCXX_SIMD_INTRINSIC constexpr void
243  __execute_on_index_sequence(_Fp&&, index_sequence<>)
244  { }
245 
246 template <size_t _Np, typename _Fp>
247  _GLIBCXX_SIMD_INTRINSIC constexpr void
248  __execute_n_times(_Fp&& __f)
249  {
250  __execute_on_index_sequence(static_cast<_Fp&&>(__f),
251  make_index_sequence<_Np>{});
252  }
253 
254 // }}}
255 // __generate_from_n_evaluations{{{
256 template <typename _R, typename _Fp, size_t... _I>
257  _GLIBCXX_SIMD_INTRINSIC constexpr _R
258  __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
259  { return _R{__f(_SizeConstant<_I>())...}; }
260 
261 template <size_t _Np, typename _R, typename _Fp>
262  _GLIBCXX_SIMD_INTRINSIC constexpr _R
263  __generate_from_n_evaluations(_Fp&& __f)
264  {
265  return __execute_on_index_sequence_with_return<_R>(
266  static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
267  }
268 
269 // }}}
270 // __call_with_n_evaluations{{{
271 template <size_t... _I, typename _F0, typename _FArgs>
272  _GLIBCXX_SIMD_INTRINSIC constexpr auto
273  __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
274  { return __f0(__fargs(_SizeConstant<_I>())...); }
275 
276 template <size_t _Np, typename _F0, typename _FArgs>
277  _GLIBCXX_SIMD_INTRINSIC constexpr auto
278  __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
279  {
280  return __call_with_n_evaluations(make_index_sequence<_Np>{},
281  static_cast<_F0&&>(__f0),
282  static_cast<_FArgs&&>(__fargs));
283  }
284 
285 // }}}
286 // __call_with_subscripts{{{
287 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
288  _GLIBCXX_SIMD_INTRINSIC constexpr auto
289  __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
290  { return __fun(__x[_First + _It]...); }
291 
292 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
293  _GLIBCXX_SIMD_INTRINSIC constexpr auto
294  __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
295  {
296  return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
297  make_index_sequence<_Np>(),
298  static_cast<_Fp&&>(__fun));
299  }
300 
301 // }}}
302 
303 // vvv ---- type traits ---- vvv
304 // integer type aliases{{{
305 using _UChar = unsigned char;
306 using _SChar = signed char;
307 using _UShort = unsigned short;
308 using _UInt = unsigned int;
309 using _ULong = unsigned long;
310 using _ULLong = unsigned long long;
311 using _LLong = long long;
312 
313 //}}}
314 // __first_of_pack{{{
315 template <typename _T0, typename...>
316  struct __first_of_pack
317  { using type = _T0; };
318 
319 template <typename... _Ts>
320  using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
321 
322 //}}}
323 // __value_type_or_identity_t {{{
324 template <typename _Tp>
325  typename _Tp::value_type
326  __value_type_or_identity_impl(int);
327 
328 template <typename _Tp>
329  _Tp
330  __value_type_or_identity_impl(float);
331 
332 template <typename _Tp>
333  using __value_type_or_identity_t
334  = decltype(__value_type_or_identity_impl<_Tp>(int()));
335 
336 // }}}
337 // __is_vectorizable {{{
338 template <typename _Tp>
339  struct __is_vectorizable : public is_arithmetic<_Tp> {};
340 
341 template <>
342  struct __is_vectorizable<bool> : public false_type {};
343 
344 template <typename _Tp>
345  inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
346 
347 // Deduces to a vectorizable type
348 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
349  using _Vectorizable = _Tp;
350 
351 // }}}
352 // _LoadStorePtr / __is_possible_loadstore_conversion {{{
353 template <typename _Ptr, typename _ValueType>
354  struct __is_possible_loadstore_conversion
355  : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
356 
357 template <>
358  struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
359 
360 // Deduces to a type allowed for load/store with the given value type.
361 template <typename _Ptr, typename _ValueType,
362  typename = enable_if_t<
363  __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
364  using _LoadStorePtr = _Ptr;
365 
366 // }}}
367 // __is_bitmask{{{
368 template <typename _Tp, typename = void_t<>>
369  struct __is_bitmask : false_type {};
370 
371 template <typename _Tp>
372  inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
373 
374 // the __mmaskXX case:
375 template <typename _Tp>
376  struct __is_bitmask<_Tp,
377  void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
378  : true_type {};
379 
380 // }}}
381 // __int_for_sizeof{{{
382 #pragma GCC diagnostic push
383 #pragma GCC diagnostic ignored "-Wpedantic"
384 template <size_t _Bytes>
385  constexpr auto
386  __int_for_sizeof()
387  {
388  if constexpr (_Bytes == sizeof(int))
389  return int();
390  #ifdef __clang__
391  else if constexpr (_Bytes == sizeof(char))
392  return char();
393  #else
394  else if constexpr (_Bytes == sizeof(_SChar))
395  return _SChar();
396  #endif
397  else if constexpr (_Bytes == sizeof(short))
398  return short();
399  #ifndef __clang__
400  else if constexpr (_Bytes == sizeof(long))
401  return long();
402  #endif
403  else if constexpr (_Bytes == sizeof(_LLong))
404  return _LLong();
405  #ifdef __SIZEOF_INT128__
406  else if constexpr (_Bytes == sizeof(__int128))
407  return __int128();
408  #endif // __SIZEOF_INT128__
409  else if constexpr (_Bytes % sizeof(int) == 0)
410  {
411  constexpr size_t _Np = _Bytes / sizeof(int);
412  struct _Ip
413  {
414  int _M_data[_Np];
415 
416  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
417  operator&(_Ip __rhs) const
418  {
419  return __generate_from_n_evaluations<_Np, _Ip>(
420  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
421  return __rhs._M_data[__i] & _M_data[__i];
422  });
423  }
424 
425  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
426  operator|(_Ip __rhs) const
427  {
428  return __generate_from_n_evaluations<_Np, _Ip>(
429  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
430  return __rhs._M_data[__i] | _M_data[__i];
431  });
432  }
433 
434  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
435  operator^(_Ip __rhs) const
436  {
437  return __generate_from_n_evaluations<_Np, _Ip>(
438  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
439  return __rhs._M_data[__i] ^ _M_data[__i];
440  });
441  }
442 
443  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
444  operator~() const
445  {
446  return __generate_from_n_evaluations<_Np, _Ip>(
447  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
448  }
449  };
450  return _Ip{};
451  }
452  else
453  static_assert(_Bytes != _Bytes, "this should be unreachable");
454  }
455 #pragma GCC diagnostic pop
456 
457 template <typename _Tp>
458  using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
459 
460 template <size_t _Np>
461  using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
462 
463 // }}}
464 // __is_fixed_size_abi{{{
465 template <typename _Tp>
466  struct __is_fixed_size_abi : false_type {};
467 
468 template <int _Np>
469  struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
470 
471 template <typename _Tp>
472  inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
473 
474 // }}}
475 // constexpr feature detection{{{
476 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
477 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
478 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
479 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
480 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
481 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
482 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
483 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
484 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
485 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
486 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
487 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
488 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
489 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
490 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
491 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
492 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
493 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
494 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
495 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
496 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
497 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
498 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
499 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
500 
501 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
502 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
503 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
504 constexpr inline bool __support_neon_float =
505 #if defined __GCC_IEC_559
506  __GCC_IEC_559 == 0;
507 #elif defined __FAST_MATH__
508  true;
509 #else
510  false;
511 #endif
512 
513 #ifdef _ARCH_PWR10
514 constexpr inline bool __have_power10vec = true;
515 #else
516 constexpr inline bool __have_power10vec = false;
517 #endif
518 #ifdef __POWER9_VECTOR__
519 constexpr inline bool __have_power9vec = true;
520 #else
521 constexpr inline bool __have_power9vec = false;
522 #endif
523 #if defined __POWER8_VECTOR__
524 constexpr inline bool __have_power8vec = true;
525 #else
526 constexpr inline bool __have_power8vec = __have_power9vec;
527 #endif
528 #if defined __VSX__
529 constexpr inline bool __have_power_vsx = true;
530 #else
531 constexpr inline bool __have_power_vsx = __have_power8vec;
532 #endif
533 #if defined __ALTIVEC__
534 constexpr inline bool __have_power_vmx = true;
535 #else
536 constexpr inline bool __have_power_vmx = __have_power_vsx;
537 #endif
538 
539 // }}}
540 // __is_scalar_abi {{{
541 template <typename _Abi>
542  constexpr bool
543  __is_scalar_abi()
544  { return is_same_v<simd_abi::scalar, _Abi>; }
545 
546 // }}}
547 // __abi_bytes_v {{{
548 template <template <int> class _Abi, int _Bytes>
549  constexpr int
550  __abi_bytes_impl(_Abi<_Bytes>*)
551  { return _Bytes; }
552 
553 template <typename _Tp>
554  constexpr int
555  __abi_bytes_impl(_Tp*)
556  { return -1; }
557 
558 template <typename _Abi>
559  inline constexpr int __abi_bytes_v
560  = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
561 
562 // }}}
563 // __is_builtin_bitmask_abi {{{
564 template <typename _Abi>
565  constexpr bool
566  __is_builtin_bitmask_abi()
567  { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
568 
569 // }}}
570 // __is_sse_abi {{{
571 template <typename _Abi>
572  constexpr bool
573  __is_sse_abi()
574  {
575  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
576  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
577  }
578 
579 // }}}
580 // __is_avx_abi {{{
581 template <typename _Abi>
582  constexpr bool
583  __is_avx_abi()
584  {
585  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
586  return _Bytes > 16 && _Bytes <= 32
587  && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
588  }
589 
590 // }}}
591 // __is_avx512_abi {{{
592 template <typename _Abi>
593  constexpr bool
594  __is_avx512_abi()
595  {
596  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
597  return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
598  }
599 
600 // }}}
601 // __is_neon_abi {{{
602 template <typename _Abi>
603  constexpr bool
604  __is_neon_abi()
605  {
606  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
607  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
608  }
609 
610 // }}}
611 // __make_dependent_t {{{
612 template <typename, typename _Up>
613  struct __make_dependent
614  { using type = _Up; };
615 
616 template <typename _Tp, typename _Up>
617  using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
618 
619 // }}}
620 // ^^^ ---- type traits ---- ^^^
621 
622 // __invoke_ub{{{
623 template <typename... _Args>
624  [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
625  __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args)
626  {
627 #ifdef _GLIBCXX_DEBUG_UB
628  __builtin_fprintf(stderr, __msg, __args...);
629  __builtin_trap();
630 #else
631  __builtin_unreachable();
632 #endif
633  }
634 
635 // }}}
636 // __assert_unreachable{{{
637 template <typename _Tp>
638  struct __assert_unreachable
639  { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
640 
641 // }}}
642 // __size_or_zero_v {{{
643 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
644  constexpr size_t
645  __size_or_zero_dispatch(int)
646  { return _Np; }
647 
648 template <typename _Tp, typename _Ap>
649  constexpr size_t
650  __size_or_zero_dispatch(float)
651  { return 0; }
652 
653 template <typename _Tp, typename _Ap>
654  inline constexpr size_t __size_or_zero_v
655  = __size_or_zero_dispatch<_Tp, _Ap>(0);
656 
657 // }}}
658 // __div_roundup {{{
659 inline constexpr size_t
660 __div_roundup(size_t __a, size_t __b)
661 { return (__a + __b - 1) / __b; }
662 
663 // }}}
664 // _ExactBool{{{
665 class _ExactBool
666 {
667  const bool _M_data;
668 
669 public:
670  _GLIBCXX_SIMD_INTRINSIC constexpr
671  _ExactBool(bool __b) : _M_data(__b) {}
672 
673  _ExactBool(int) = delete;
674 
675  _GLIBCXX_SIMD_INTRINSIC constexpr
676  operator bool() const
677  { return _M_data; }
678 };
679 
680 // }}}
681 // __may_alias{{{
682 /**@internal
683  * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
684  * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
685  * that support it).
686  */
687 template <typename _Tp>
688  using __may_alias [[__gnu__::__may_alias__]] = _Tp;
689 
690 // }}}
691 // _UnsupportedBase {{{
692 // simd and simd_mask base for unsupported <_Tp, _Abi>
693 struct _UnsupportedBase
694 {
695  _UnsupportedBase() = delete;
696  _UnsupportedBase(const _UnsupportedBase&) = delete;
697  _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
698  ~_UnsupportedBase() = delete;
699 };
700 
701 // }}}
702 // _InvalidTraits {{{
703 /**
704  * @internal
705  * Defines the implementation of __a given <_Tp, _Abi>.
706  *
707  * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
708  * possible. Static assertions in the type definition do not suffice. It is
709  * important that SFINAE works.
710  */
711 struct _InvalidTraits
712 {
713  using _IsValid = false_type;
714  using _SimdBase = _UnsupportedBase;
715  using _MaskBase = _UnsupportedBase;
716 
717  static constexpr size_t _S_full_size = 0;
718  static constexpr bool _S_is_partial = false;
719 
720  static constexpr size_t _S_simd_align = 1;
721  struct _SimdImpl;
722  struct _SimdMember {};
723  struct _SimdCastType;
724 
725  static constexpr size_t _S_mask_align = 1;
726  struct _MaskImpl;
727  struct _MaskMember {};
728  struct _MaskCastType;
729 };
730 
731 // }}}
732 // _SimdTraits {{{
733 template <typename _Tp, typename _Abi, typename = void_t<>>
734  struct _SimdTraits : _InvalidTraits {};
735 
736 // }}}
737 // __private_init, __bitset_init{{{
738 /**
739  * @internal
740  * Tag used for private init constructor of simd and simd_mask
741  */
742 inline constexpr struct _PrivateInit {} __private_init = {};
743 
744 inline constexpr struct _BitsetInit {} __bitset_init = {};
745 
746 // }}}
747 // __is_narrowing_conversion<_From, _To>{{{
748 template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
749  bool = is_arithmetic_v<_To>>
750  struct __is_narrowing_conversion;
751 
752 // ignore "signed/unsigned mismatch" in the following trait.
753 // The implicit conversions will do the right thing here.
754 template <typename _From, typename _To>
755  struct __is_narrowing_conversion<_From, _To, true, true>
756  : public __bool_constant<(
757  __digits_v<_From> > __digits_v<_To>
758  || __finite_max_v<_From> > __finite_max_v<_To>
759  || __finite_min_v<_From> < __finite_min_v<_To>
760  || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
761 
762 template <typename _Tp>
763  struct __is_narrowing_conversion<_Tp, bool, true, true>
764  : public true_type {};
765 
766 template <>
767  struct __is_narrowing_conversion<bool, bool, true, true>
768  : public false_type {};
769 
770 template <typename _Tp>
771  struct __is_narrowing_conversion<_Tp, _Tp, true, true>
772  : public false_type {};
773 
774 template <typename _From, typename _To>
775  struct __is_narrowing_conversion<_From, _To, false, true>
776  : public negation<is_convertible<_From, _To>> {};
777 
778 // }}}
779 // __converts_to_higher_integer_rank{{{
780 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
781  struct __converts_to_higher_integer_rank : public true_type {};
782 
783 // this may fail for char -> short if sizeof(char) == sizeof(short)
784 template <typename _From, typename _To>
785  struct __converts_to_higher_integer_rank<_From, _To, false>
786  : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
787 
788 // }}}
789 // __data(simd/simd_mask) {{{
790 template <typename _Tp, typename _Ap>
791  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
792  __data(const simd<_Tp, _Ap>& __x);
793 
794 template <typename _Tp, typename _Ap>
795  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
796  __data(simd<_Tp, _Ap>& __x);
797 
798 template <typename _Tp, typename _Ap>
799  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
800  __data(const simd_mask<_Tp, _Ap>& __x);
801 
802 template <typename _Tp, typename _Ap>
803  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
804  __data(simd_mask<_Tp, _Ap>& __x);
805 
806 // }}}
807 // _SimdConverter {{{
808 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
809  typename = void>
810  struct _SimdConverter;
811 
812 template <typename _Tp, typename _Ap>
813  struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
814  {
815  template <typename _Up>
816  _GLIBCXX_SIMD_INTRINSIC const _Up&
817  operator()(const _Up& __x)
818  { return __x; }
819  };
820 
821 // }}}
822 // __to_value_type_or_member_type {{{
823 template <typename _V>
824  _GLIBCXX_SIMD_INTRINSIC constexpr auto
825  __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
826  { return __data(__x); }
827 
828 template <typename _V>
829  _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
830  __to_value_type_or_member_type(const typename _V::value_type& __x)
831  { return __x; }
832 
833 // }}}
834 // __bool_storage_member_type{{{
835 template <size_t _Size>
836  struct __bool_storage_member_type;
837 
838 template <size_t _Size>
839  using __bool_storage_member_type_t =
840  typename __bool_storage_member_type<_Size>::type;
841 
842 // }}}
843 // _SimdTuple {{{
844 // why not tuple?
845 // 1. tuple gives no guarantee about the storage order, but I require
846 // storage
847 // equivalent to array<_Tp, _Np>
848 // 2. direct access to the element type (first template argument)
849 // 3. enforces equal element type, only different _Abi types are allowed
850 template <typename _Tp, typename... _Abis>
851  struct _SimdTuple;
852 
853 //}}}
854 // __fixed_size_storage_t {{{
855 template <typename _Tp, int _Np>
856  struct __fixed_size_storage;
857 
858 template <typename _Tp, int _Np>
859  using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
860 
861 // }}}
862 // _SimdWrapper fwd decl{{{
863 template <typename _Tp, size_t _Size, typename = void_t<>>
864  struct _SimdWrapper;
865 
866 template <typename _Tp>
867  using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
868 template <typename _Tp>
869  using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
870 template <typename _Tp>
871  using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
872 template <typename _Tp>
873  using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
874 
875 // }}}
876 // __is_simd_wrapper {{{
877 template <typename _Tp>
878  struct __is_simd_wrapper : false_type {};
879 
880 template <typename _Tp, size_t _Np>
881  struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
882 
883 template <typename _Tp>
884  inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
885 
886 // }}}
887 // _BitOps {{{
888 struct _BitOps
889 {
890  // _S_bit_iteration {{{
891  template <typename _Tp, typename _Fp>
892  static void
893  _S_bit_iteration(_Tp __mask, _Fp&& __f)
894  {
895  static_assert(sizeof(_ULLong) >= sizeof(_Tp));
896  conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
897  if constexpr (is_convertible_v<_Tp, decltype(__k)>)
898  __k = __mask;
899  else
900  __k = __mask.to_ullong();
901  while(__k)
902  {
903  __f(std::__countr_zero(__k));
904  __k &= (__k - 1);
905  }
906  }
907 
908  //}}}
909 };
910 
911 //}}}
912 // __increment, __decrement {{{
913 template <typename _Tp = void>
914  struct __increment
915  { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
916 
917 template <>
918  struct __increment<void>
919  {
920  template <typename _Tp>
921  constexpr _Tp
922  operator()(_Tp __a) const
923  { return ++__a; }
924  };
925 
926 template <typename _Tp = void>
927  struct __decrement
928  { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
929 
930 template <>
931  struct __decrement<void>
932  {
933  template <typename _Tp>
934  constexpr _Tp
935  operator()(_Tp __a) const
936  { return --__a; }
937  };
938 
939 // }}}
940 // _ValuePreserving(OrInt) {{{
941 template <typename _From, typename _To,
942  typename = enable_if_t<negation<
943  __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
944  using _ValuePreserving = _From;
945 
946 template <typename _From, typename _To,
947  typename _DecayedFrom = __remove_cvref_t<_From>,
948  typename = enable_if_t<conjunction<
949  is_convertible<_From, _To>,
950  disjunction<
951  is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
952  conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
953  negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
954  using _ValuePreservingOrInt = _From;
955 
956 // }}}
957 // __intrinsic_type {{{
958 template <typename _Tp, size_t _Bytes, typename = void_t<>>
959  struct __intrinsic_type;
960 
961 template <typename _Tp, size_t _Size>
962  using __intrinsic_type_t =
963  typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
964 
965 template <typename _Tp>
966  using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
967 template <typename _Tp>
968  using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
969 template <typename _Tp>
970  using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
971 template <typename _Tp>
972  using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
973 template <typename _Tp>
974  using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
975 template <typename _Tp>
976  using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
977 
978 // }}}
979 // _BitMask {{{
980 template <size_t _Np, bool _Sanitized = false>
981  struct _BitMask;
982 
983 template <size_t _Np, bool _Sanitized>
984  struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
985 
986 template <size_t _Np>
987  using _SanitizedBitMask = _BitMask<_Np, true>;
988 
989 template <size_t _Np, bool _Sanitized>
990  struct _BitMask
991  {
992  static_assert(_Np > 0);
993 
994  static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
995 
996  using _Tp = conditional_t<_Np == 1, bool,
997  make_unsigned_t<__int_with_sizeof_t<std::min(
998  sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
999 
1000  static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1001 
1002  _Tp _M_bits[_S_array_size];
1003 
1004  static constexpr int _S_unused_bits
1005  = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1006 
1007  static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1008 
1009  constexpr _BitMask() noexcept = default;
1010 
1011  constexpr _BitMask(unsigned long long __x) noexcept
1012  : _M_bits{static_cast<_Tp>(__x)} {}
1013 
1014  _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1015 
1016  constexpr _BitMask(const _BitMask&) noexcept = default;
1017 
1018  template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1019  && _Sanitized == true>>
1020  constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1021  : _BitMask(__rhs._M_sanitized()) {}
1022 
1023  constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1024  {
1025  static_assert(_S_array_size == 1);
1026  return _M_bits[0];
1027  }
1028 
1029  // precondition: is sanitized
1030  constexpr _Tp
1031  _M_to_bits() const noexcept
1032  {
1033  static_assert(_S_array_size == 1);
1034  return _M_bits[0];
1035  }
1036 
1037  // precondition: is sanitized
1038  constexpr unsigned long long
1039  to_ullong() const noexcept
1040  {
1041  static_assert(_S_array_size == 1);
1042  return _M_bits[0];
1043  }
1044 
1045  // precondition: is sanitized
1046  constexpr unsigned long
1047  to_ulong() const noexcept
1048  {
1049  static_assert(_S_array_size == 1);
1050  return _M_bits[0];
1051  }
1052 
1053  constexpr bitset<_Np>
1054  _M_to_bitset() const noexcept
1055  {
1056  static_assert(_S_array_size == 1);
1057  return _M_bits[0];
1058  }
1059 
1060  constexpr decltype(auto)
1061  _M_sanitized() const noexcept
1062  {
1063  if constexpr (_Sanitized)
1064  return *this;
1065  else if constexpr (_Np == 1)
1066  return _SanitizedBitMask<_Np>(_M_bits[0]);
1067  else
1068  {
1069  _SanitizedBitMask<_Np> __r = {};
1070  for (int __i = 0; __i < _S_array_size; ++__i)
1071  __r._M_bits[__i] = _M_bits[__i];
1072  if constexpr (_S_unused_bits > 0)
1073  __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1074  return __r;
1075  }
1076  }
1077 
1078  template <size_t _Mp, bool _LSanitized>
1079  constexpr _BitMask<_Np + _Mp, _Sanitized>
1080  _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1081  {
1082  constexpr size_t _RN = _Np + _Mp;
1083  using _Rp = _BitMask<_RN, _Sanitized>;
1084  if constexpr (_Rp::_S_array_size == 1)
1085  {
1086  _Rp __r{{_M_bits[0]}};
1087  __r._M_bits[0] <<= _Mp;
1088  __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1089  return __r;
1090  }
1091  else
1092  __assert_unreachable<_Rp>();
1093  }
1094 
1095  // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1096  // significant bits. If the operation implicitly produces a sanitized bitmask,
1097  // the result type will have _Sanitized set.
1098  template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1099  constexpr auto
1100  _M_extract() const noexcept
1101  {
1102  static_assert(_Np > _DropLsb);
1103  static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1104  "not implemented for bitmasks larger than one ullong");
1105  if constexpr (_NewSize == 1)
1106  // must sanitize because the return _Tp is bool
1107  return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1108  else
1109  return _BitMask<_NewSize,
1110  ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1111  && _NewSize + _DropLsb <= _Np)
1112  || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1113  && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1114  >> _DropLsb);
1115  }
1116 
1117  // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1118  constexpr bool
1119  all() const noexcept
1120  {
1121  if constexpr (_Np == 1)
1122  return _M_bits[0];
1123  else if constexpr (!_Sanitized)
1124  return _M_sanitized().all();
1125  else
1126  {
1127  constexpr _Tp __allbits = ~_Tp();
1128  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1129  if (_M_bits[__i] != __allbits)
1130  return false;
1131  return _M_bits[_S_array_size - 1] == _S_bitmask;
1132  }
1133  }
1134 
1135  // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1136  // false.
1137  constexpr bool
1138  any() const noexcept
1139  {
1140  if constexpr (_Np == 1)
1141  return _M_bits[0];
1142  else if constexpr (!_Sanitized)
1143  return _M_sanitized().any();
1144  else
1145  {
1146  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1147  if (_M_bits[__i] != 0)
1148  return true;
1149  return _M_bits[_S_array_size - 1] != 0;
1150  }
1151  }
1152 
1153  // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1154  constexpr bool
1155  none() const noexcept
1156  {
1157  if constexpr (_Np == 1)
1158  return !_M_bits[0];
1159  else if constexpr (!_Sanitized)
1160  return _M_sanitized().none();
1161  else
1162  {
1163  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1164  if (_M_bits[__i] != 0)
1165  return false;
1166  return _M_bits[_S_array_size - 1] == 0;
1167  }
1168  }
1169 
1170  // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1171  // false.
1172  constexpr int
1173  count() const noexcept
1174  {
1175  if constexpr (_Np == 1)
1176  return _M_bits[0];
1177  else if constexpr (!_Sanitized)
1178  return _M_sanitized().none();
1179  else
1180  {
1181  int __result = __builtin_popcountll(_M_bits[0]);
1182  for (int __i = 1; __i < _S_array_size; ++__i)
1183  __result += __builtin_popcountll(_M_bits[__i]);
1184  return __result;
1185  }
1186  }
1187 
1188  // Returns the bit at offset __i as bool.
1189  constexpr bool
1190  operator[](size_t __i) const noexcept
1191  {
1192  if constexpr (_Np == 1)
1193  return _M_bits[0];
1194  else if constexpr (_S_array_size == 1)
1195  return (_M_bits[0] >> __i) & 1;
1196  else
1197  {
1198  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1199  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1200  return (_M_bits[__j] >> __shift) & 1;
1201  }
1202  }
1203 
1204  template <size_t __i>
1205  constexpr bool
1206  operator[](_SizeConstant<__i>) const noexcept
1207  {
1208  static_assert(__i < _Np);
1209  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1210  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1211  return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1212  }
1213 
1214  // Set the bit at offset __i to __x.
1215  constexpr void
1216  set(size_t __i, bool __x) noexcept
1217  {
1218  if constexpr (_Np == 1)
1219  _M_bits[0] = __x;
1220  else if constexpr (_S_array_size == 1)
1221  {
1222  _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1223  _M_bits[0] |= _Tp(_Tp(__x) << __i);
1224  }
1225  else
1226  {
1227  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1228  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1229  _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1230  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1231  }
1232  }
1233 
1234  template <size_t __i>
1235  constexpr void
1236  set(_SizeConstant<__i>, bool __x) noexcept
1237  {
1238  static_assert(__i < _Np);
1239  if constexpr (_Np == 1)
1240  _M_bits[0] = __x;
1241  else
1242  {
1243  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1244  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1245  constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1246  _M_bits[__j] &= __mask;
1247  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1248  }
1249  }
1250 
1251  // Inverts all bits. Sanitized input leads to sanitized output.
1252  constexpr _BitMask
1253  operator~() const noexcept
1254  {
1255  if constexpr (_Np == 1)
1256  return !_M_bits[0];
1257  else
1258  {
1259  _BitMask __result{};
1260  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1261  __result._M_bits[__i] = ~_M_bits[__i];
1262  if constexpr (_Sanitized)
1263  __result._M_bits[_S_array_size - 1]
1264  = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1265  else
1266  __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1267  return __result;
1268  }
1269  }
1270 
1271  constexpr _BitMask&
1272  operator^=(const _BitMask& __b) & noexcept
1273  {
1274  __execute_n_times<_S_array_size>(
1275  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1276  return *this;
1277  }
1278 
1279  constexpr _BitMask&
1280  operator|=(const _BitMask& __b) & noexcept
1281  {
1282  __execute_n_times<_S_array_size>(
1283  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1284  return *this;
1285  }
1286 
1287  constexpr _BitMask&
1288  operator&=(const _BitMask& __b) & noexcept
1289  {
1290  __execute_n_times<_S_array_size>(
1291  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1292  return *this;
1293  }
1294 
1295  friend constexpr _BitMask
1296  operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1297  {
1298  _BitMask __r = __a;
1299  __r ^= __b;
1300  return __r;
1301  }
1302 
1303  friend constexpr _BitMask
1304  operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1305  {
1306  _BitMask __r = __a;
1307  __r |= __b;
1308  return __r;
1309  }
1310 
1311  friend constexpr _BitMask
1312  operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1313  {
1314  _BitMask __r = __a;
1315  __r &= __b;
1316  return __r;
1317  }
1318 
1319  _GLIBCXX_SIMD_INTRINSIC
1320  constexpr bool
1321  _M_is_constprop() const
1322  {
1323  if constexpr (_S_array_size == 0)
1324  return __builtin_constant_p(_M_bits[0]);
1325  else
1326  {
1327  for (int __i = 0; __i < _S_array_size; ++__i)
1328  if (!__builtin_constant_p(_M_bits[__i]))
1329  return false;
1330  return true;
1331  }
1332  }
1333  };
1334 
1335 // }}}
1336 
1337 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1338 // __min_vector_size {{{
1339 template <typename _Tp = void>
1340  static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1341 
1342 #if _GLIBCXX_SIMD_HAVE_NEON
1343 template <>
1344  inline constexpr int __min_vector_size<void> = 8;
1345 #else
1346 template <>
1347  inline constexpr int __min_vector_size<void> = 16;
1348 #endif
1349 
1350 // }}}
1351 // __vector_type {{{
1352 template <typename _Tp, size_t _Np, typename = void>
1353  struct __vector_type_n {};
1354 
1355 // substition failure for 0-element case
1356 template <typename _Tp>
1357  struct __vector_type_n<_Tp, 0, void> {};
1358 
1359 // special case 1-element to be _Tp itself
1360 template <typename _Tp>
1361  struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1362  { using type = _Tp; };
1363 
1364 // else, use GNU-style builtin vector types
1365 template <typename _Tp, size_t _Np>
1366  struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1367  {
1368  static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1369 
1370  static constexpr size_t _S_Bytes =
1371 #ifdef __i386__
1372  // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1373  // those objects are passed via MMX registers and nothing ever calls EMMS.
1374  _S_Np2 == 8 ? 16 :
1375 #endif
1376  _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1377  : _S_Np2;
1378 
1379  using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1380  };
1381 
1382 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1383  struct __vector_type;
1384 
1385 template <typename _Tp, size_t _Bytes>
1386  struct __vector_type<_Tp, _Bytes, 0>
1387  : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1388 
1389 template <typename _Tp, size_t _Size>
1390  using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1391 
1392 template <typename _Tp>
1393  using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1394 template <typename _Tp>
1395  using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1396 template <typename _Tp>
1397  using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1398 template <typename _Tp>
1399  using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1400 template <typename _Tp>
1401  using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1402 template <typename _Tp>
1403  using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1404 
1405 // }}}
1406 // __is_vector_type {{{
1407 template <typename _Tp, typename = void_t<>>
1408  struct __is_vector_type : false_type {};
1409 
1410 template <typename _Tp>
1411  struct __is_vector_type<
1412  _Tp, void_t<typename __vector_type<
1413  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1414  : is_same<_Tp, typename __vector_type<
1415  remove_reference_t<decltype(declval<_Tp>()[0])>,
1416  sizeof(_Tp)>::type> {};
1417 
1418 template <typename _Tp>
1419  inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1420 
1421 // }}}
1422 // __is_intrinsic_type {{{
1423 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1424 template <typename _Tp>
1425  using __is_intrinsic_type = __is_vector_type<_Tp>;
1426 #else // not SSE (x86)
1427 template <typename _Tp, typename = void_t<>>
1428  struct __is_intrinsic_type : false_type {};
1429 
1430 template <typename _Tp>
1431  struct __is_intrinsic_type<
1432  _Tp, void_t<typename __intrinsic_type<
1433  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1434  : is_same<_Tp, typename __intrinsic_type<
1435  remove_reference_t<decltype(declval<_Tp>()[0])>,
1436  sizeof(_Tp)>::type> {};
1437 #endif
1438 
1439 template <typename _Tp>
1440  inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1441 
1442 // }}}
1443 // _VectorTraits{{{
1444 template <typename _Tp, typename = void_t<>>
1445  struct _VectorTraitsImpl;
1446 
1447 template <typename _Tp>
1448  struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1449  || __is_intrinsic_type_v<_Tp>>>
1450  {
1451  using type = _Tp;
1452  using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1453  static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1454  using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1455  template <typename _Up, int _W = _S_full_size>
1456  static constexpr bool _S_is
1457  = is_same_v<value_type, _Up> && _W == _S_full_size;
1458  };
1459 
1460 template <typename _Tp, size_t _Np>
1461  struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1462  void_t<__vector_type_t<_Tp, _Np>>>
1463  {
1464  using type = __vector_type_t<_Tp, _Np>;
1465  using value_type = _Tp;
1466  static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1467  using _Wrapper = _SimdWrapper<_Tp, _Np>;
1468  static constexpr bool _S_is_partial = (_Np == _S_full_size);
1469  static constexpr int _S_partial_width = _Np;
1470  template <typename _Up, int _W = _S_full_size>
1471  static constexpr bool _S_is
1472  = is_same_v<value_type, _Up>&& _W == _S_full_size;
1473  };
1474 
1475 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1476  using _VectorTraits = _VectorTraitsImpl<_Tp>;
1477 
1478 // }}}
1479 // __as_vector{{{
1480 template <typename _V>
1481  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1482  __as_vector(_V __x)
1483  {
1484  if constexpr (__is_vector_type_v<_V>)
1485  return __x;
1486  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1487  return __data(__x)._M_data;
1488  else if constexpr (__is_vectorizable_v<_V>)
1489  return __vector_type_t<_V, 2>{__x};
1490  else
1491  return __x._M_data;
1492  }
1493 
1494 // }}}
1495 // __as_wrapper{{{
1496 template <size_t _Np = 0, typename _V>
1497  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1498  __as_wrapper(_V __x)
1499  {
1500  if constexpr (__is_vector_type_v<_V>)
1501  return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1502  (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1503  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1504  {
1505  static_assert(_V::size() == _Np);
1506  return __data(__x);
1507  }
1508  else
1509  {
1510  static_assert(_V::_S_size == _Np);
1511  return __x;
1512  }
1513  }
1514 
1515 // }}}
1516 // __intrin_bitcast{{{
1517 template <typename _To, typename _From>
1518  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1519  __intrin_bitcast(_From __v)
1520  {
1521  static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1522  && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1523  if constexpr (sizeof(_To) == sizeof(_From))
1524  return reinterpret_cast<_To>(__v);
1525  else if constexpr (sizeof(_From) > sizeof(_To))
1526  if constexpr (sizeof(_To) >= 16)
1527  return reinterpret_cast<const __may_alias<_To>&>(__v);
1528  else
1529  {
1530  _To __r;
1531  __builtin_memcpy(&__r, &__v, sizeof(_To));
1532  return __r;
1533  }
1534 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1535  else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1536  return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1537  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1538  else if constexpr (__have_avx512f && sizeof(_From) == 16
1539  && sizeof(_To) == 64)
1540  return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1541  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1542  else if constexpr (__have_avx512f && sizeof(_From) == 32
1543  && sizeof(_To) == 64)
1544  return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1545  reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1546 #endif // _GLIBCXX_SIMD_X86INTRIN
1547  else if constexpr (sizeof(__v) <= 8)
1548  return reinterpret_cast<_To>(
1549  __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1550  reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1551  else
1552  {
1553  static_assert(sizeof(_To) > sizeof(_From));
1554  _To __r = {};
1555  __builtin_memcpy(&__r, &__v, sizeof(_From));
1556  return __r;
1557  }
1558  }
1559 
1560 // }}}
1561 // __vector_bitcast{{{
1562 template <typename _To, size_t _NN = 0, typename _From,
1563  typename _FromVT = _VectorTraits<_From>,
1564  size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1565  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1566  __vector_bitcast(_From __x)
1567  {
1568  using _R = __vector_type_t<_To, _Np>;
1569  return __intrin_bitcast<_R>(__x);
1570  }
1571 
1572 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1573  size_t _Np
1574  = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1575  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1576  __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1577  {
1578  static_assert(_Np > 1);
1579  return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1580  }
1581 
1582 // }}}
1583 // __convert_x86 declarations {{{
1584 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1585 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1586  _To __convert_x86(_Tp);
1587 
1588 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1589  _To __convert_x86(_Tp, _Tp);
1590 
1591 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1592  _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1593 
1594 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1595  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1596 
1597 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1598  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1599  _Tp, _Tp, _Tp, _Tp);
1600 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1601 
1602 //}}}
1603 // __bit_cast {{{
1604 template <typename _To, typename _From>
1605  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1606  __bit_cast(const _From __x)
1607  {
1608  // TODO: implement with / replace by __builtin_bit_cast ASAP
1609  static_assert(sizeof(_To) == sizeof(_From));
1610  constexpr bool __to_is_vectorizable
1611  = is_arithmetic_v<_To> || is_enum_v<_To>;
1612  constexpr bool __from_is_vectorizable
1613  = is_arithmetic_v<_From> || is_enum_v<_From>;
1614  if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1615  return reinterpret_cast<_To>(__x);
1616  else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1617  {
1618  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1619  return reinterpret_cast<_To>(_FV{__x});
1620  }
1621  else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1622  {
1623  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1624  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1625  return reinterpret_cast<_TV>(_FV{__x})[0];
1626  }
1627  else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1628  {
1629  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1630  return reinterpret_cast<_TV>(__x)[0];
1631  }
1632  else
1633  {
1634  _To __r;
1635  __builtin_memcpy(reinterpret_cast<char*>(&__r),
1636  reinterpret_cast<const char*>(&__x), sizeof(_To));
1637  return __r;
1638  }
1639  }
1640 
1641 // }}}
1642 // __to_intrin {{{
1643 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1644  typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1645  _GLIBCXX_SIMD_INTRINSIC constexpr _R
1646  __to_intrin(_Tp __x)
1647  {
1648  static_assert(sizeof(__x) <= sizeof(_R),
1649  "__to_intrin may never drop values off the end");
1650  if constexpr (sizeof(__x) == sizeof(_R))
1651  return reinterpret_cast<_R>(__as_vector(__x));
1652  else
1653  {
1654  using _Up = __int_for_sizeof_t<_Tp>;
1655  return reinterpret_cast<_R>(
1656  __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1657  }
1658  }
1659 
1660 // }}}
1661 // __make_vector{{{
1662 template <typename _Tp, typename... _Args>
1663  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1664  __make_vector(const _Args&... __args)
1665  { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; }
1666 
1667 // }}}
1668 // __vector_broadcast{{{
1669 template <size_t _Np, typename _Tp, size_t... _I>
1670  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1671  __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1672  { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1673 
1674 template <size_t _Np, typename _Tp>
1675  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1676  __vector_broadcast(_Tp __x)
1677  { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1678 
1679 // }}}
1680 // __generate_vector{{{
1681  template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1682  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1683  __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1684  { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; }
1685 
1686 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1687  _GLIBCXX_SIMD_INTRINSIC constexpr _V
1688  __generate_vector(_Gp&& __gen)
1689  {
1690  if constexpr (__is_vector_type_v<_V>)
1691  return __generate_vector_impl<typename _VVT::value_type,
1692  _VVT::_S_full_size>(
1693  static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1694  else
1695  return __generate_vector_impl<typename _VVT::value_type,
1696  _VVT::_S_partial_width>(
1697  static_cast<_Gp&&>(__gen),
1698  make_index_sequence<_VVT::_S_partial_width>());
1699  }
1700 
1701 template <typename _Tp, size_t _Np, typename _Gp>
1702  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1703  __generate_vector(_Gp&& __gen)
1704  {
1705  return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1706  make_index_sequence<_Np>());
1707  }
1708 
1709 // }}}
1710 // __xor{{{
1711 template <typename _TW>
1712  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1713  __xor(_TW __a, _TW __b) noexcept
1714  {
1715  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1716  {
1717  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1718  _VectorTraitsImpl<_TW>>::value_type;
1719  if constexpr (is_floating_point_v<_Tp>)
1720  {
1721  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1722  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1723  ^ __vector_bitcast<_Ip>(__b));
1724  }
1725  else if constexpr (__is_vector_type_v<_TW>)
1726  return __a ^ __b;
1727  else
1728  return __a._M_data ^ __b._M_data;
1729  }
1730  else
1731  return __a ^ __b;
1732  }
1733 
1734 // }}}
1735 // __or{{{
1736 template <typename _TW>
1737  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1738  __or(_TW __a, _TW __b) noexcept
1739  {
1740  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1741  {
1742  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1743  _VectorTraitsImpl<_TW>>::value_type;
1744  if constexpr (is_floating_point_v<_Tp>)
1745  {
1746  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1747  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1748  | __vector_bitcast<_Ip>(__b));
1749  }
1750  else if constexpr (__is_vector_type_v<_TW>)
1751  return __a | __b;
1752  else
1753  return __a._M_data | __b._M_data;
1754  }
1755  else
1756  return __a | __b;
1757  }
1758 
1759 // }}}
1760 // __and{{{
1761 template <typename _TW>
1762  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1763  __and(_TW __a, _TW __b) noexcept
1764  {
1765  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1766  {
1767  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1768  _VectorTraitsImpl<_TW>>::value_type;
1769  if constexpr (is_floating_point_v<_Tp>)
1770  {
1771  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1772  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1773  & __vector_bitcast<_Ip>(__b));
1774  }
1775  else if constexpr (__is_vector_type_v<_TW>)
1776  return __a & __b;
1777  else
1778  return __a._M_data & __b._M_data;
1779  }
1780  else
1781  return __a & __b;
1782  }
1783 
1784 // }}}
1785 // __andnot{{{
1786 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1787 static constexpr struct
1788 {
1789  _GLIBCXX_SIMD_INTRINSIC __v4sf
1790  operator()(__v4sf __a, __v4sf __b) const noexcept
1791  { return __builtin_ia32_andnps(__a, __b); }
1792 
1793  _GLIBCXX_SIMD_INTRINSIC __v2df
1794  operator()(__v2df __a, __v2df __b) const noexcept
1795  { return __builtin_ia32_andnpd(__a, __b); }
1796 
1797  _GLIBCXX_SIMD_INTRINSIC __v2di
1798  operator()(__v2di __a, __v2di __b) const noexcept
1799  { return __builtin_ia32_pandn128(__a, __b); }
1800 
1801  _GLIBCXX_SIMD_INTRINSIC __v8sf
1802  operator()(__v8sf __a, __v8sf __b) const noexcept
1803  { return __builtin_ia32_andnps256(__a, __b); }
1804 
1805  _GLIBCXX_SIMD_INTRINSIC __v4df
1806  operator()(__v4df __a, __v4df __b) const noexcept
1807  { return __builtin_ia32_andnpd256(__a, __b); }
1808 
1809  _GLIBCXX_SIMD_INTRINSIC __v4di
1810  operator()(__v4di __a, __v4di __b) const noexcept
1811  {
1812  if constexpr (__have_avx2)
1813  return __builtin_ia32_andnotsi256(__a, __b);
1814  else
1815  return reinterpret_cast<__v4di>(
1816  __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1817  reinterpret_cast<__v4df>(__b)));
1818  }
1819 
1820  _GLIBCXX_SIMD_INTRINSIC __v16sf
1821  operator()(__v16sf __a, __v16sf __b) const noexcept
1822  {
1823  if constexpr (__have_avx512dq)
1824  return _mm512_andnot_ps(__a, __b);
1825  else
1826  return reinterpret_cast<__v16sf>(
1827  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1828  reinterpret_cast<__v8di>(__b)));
1829  }
1830 
1831  _GLIBCXX_SIMD_INTRINSIC __v8df
1832  operator()(__v8df __a, __v8df __b) const noexcept
1833  {
1834  if constexpr (__have_avx512dq)
1835  return _mm512_andnot_pd(__a, __b);
1836  else
1837  return reinterpret_cast<__v8df>(
1838  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1839  reinterpret_cast<__v8di>(__b)));
1840  }
1841 
1842  _GLIBCXX_SIMD_INTRINSIC __v8di
1843  operator()(__v8di __a, __v8di __b) const noexcept
1844  { return _mm512_andnot_si512(__a, __b); }
1845 } _S_x86_andnot;
1846 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1847 
1848 template <typename _TW>
1849  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1850  __andnot(_TW __a, _TW __b) noexcept
1851  {
1852  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1853  {
1854  using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1855  _VectorTraitsImpl<_TW>>;
1856  using _Tp = typename _TVT::value_type;
1857 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1858  if constexpr (sizeof(_TW) >= 16)
1859  {
1860  const auto __ai = __to_intrin(__a);
1861  const auto __bi = __to_intrin(__b);
1862  if (!__builtin_is_constant_evaluated()
1863  && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1864  {
1865  const auto __r = _S_x86_andnot(__ai, __bi);
1866  if constexpr (is_convertible_v<decltype(__r), _TW>)
1867  return __r;
1868  else
1869  return reinterpret_cast<typename _TVT::type>(__r);
1870  }
1871  }
1872 #endif // _GLIBCXX_SIMD_X86INTRIN
1873  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1874  return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1875  & __vector_bitcast<_Ip>(__b));
1876  }
1877  else
1878  return ~__a & __b;
1879  }
1880 
1881 // }}}
1882 // __not{{{
1883 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1884  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1885  __not(_Tp __a) noexcept
1886  {
1887  if constexpr (is_floating_point_v<typename _TVT::value_type>)
1888  return reinterpret_cast<typename _TVT::type>(
1889  ~__vector_bitcast<unsigned>(__a));
1890  else
1891  return ~__a;
1892  }
1893 
1894 // }}}
1895 // __concat{{{
1896 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1897  typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>>
1898  constexpr _R
1899  __concat(_Tp a_, _Tp b_)
1900  {
1901 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1902  using _W
1903  = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1904  conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1905  long long, typename _TVT::value_type>>;
1906  constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1907  const auto __a = __vector_bitcast<_W>(a_);
1908  const auto __b = __vector_bitcast<_W>(b_);
1909  using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1910 #else
1911  constexpr int input_width = _TVT::_S_full_size;
1912  const _Tp& __a = a_;
1913  const _Tp& __b = b_;
1914  using _Up = _R;
1915 #endif
1916  if constexpr (input_width == 2)
1917  return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1918  else if constexpr (input_width == 4)
1919  return reinterpret_cast<_R>(
1920  _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1921  else if constexpr (input_width == 8)
1922  return reinterpret_cast<_R>(
1923  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1924  __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1925  else if constexpr (input_width == 16)
1926  return reinterpret_cast<_R>(
1927  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1928  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1929  __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1930  __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1931  __b[12], __b[13], __b[14], __b[15]});
1932  else if constexpr (input_width == 32)
1933  return reinterpret_cast<_R>(
1934  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1935  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1936  __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1937  __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1938  __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1939  __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1940  __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1941  __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1942  __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1943  __b[31]});
1944  }
1945 
1946 // }}}
1947 // __zero_extend {{{
1948 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1949  struct _ZeroExtendProxy
1950  {
1951  using value_type = typename _TVT::value_type;
1952  static constexpr size_t _Np = _TVT::_S_full_size;
1953  const _Tp __x;
1954 
1955  template <typename _To, typename _ToVT = _VectorTraits<_To>,
1956  typename
1957  = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1958  _GLIBCXX_SIMD_INTRINSIC operator _To() const
1959  {
1960  constexpr size_t _ToN = _ToVT::_S_full_size;
1961  if constexpr (_ToN == _Np)
1962  return __x;
1963  else if constexpr (_ToN == 2 * _Np)
1964  {
1965 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1966  if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1967  return __vector_bitcast<value_type>(
1968  _mm256_insertf128_ps(__m256(), __x, 0));
1969  else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1970  return __vector_bitcast<value_type>(
1971  _mm256_insertf128_pd(__m256d(), __x, 0));
1972  else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1973  return __vector_bitcast<value_type>(
1974  _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1975  else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1976  {
1977  if constexpr (__have_avx512dq)
1978  return __vector_bitcast<value_type>(
1979  _mm512_insertf32x8(__m512(), __x, 0));
1980  else
1981  return reinterpret_cast<__m512>(
1982  _mm512_insertf64x4(__m512d(),
1983  reinterpret_cast<__m256d>(__x), 0));
1984  }
1985  else if constexpr (__have_avx512f
1986  && _TVT::template _S_is<double, 4>)
1987  return __vector_bitcast<value_type>(
1988  _mm512_insertf64x4(__m512d(), __x, 0));
1989  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1990  return __vector_bitcast<value_type>(
1991  _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1992 #endif
1993  return __concat(__x, _Tp());
1994  }
1995  else if constexpr (_ToN == 4 * _Np)
1996  {
1997 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1998  if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
1999  {
2000  return __vector_bitcast<value_type>(
2001  _mm512_insertf64x2(__m512d(), __x, 0));
2002  }
2003  else if constexpr (__have_avx512f
2004  && is_floating_point_v<value_type>)
2005  {
2006  return __vector_bitcast<value_type>(
2007  _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2008  0));
2009  }
2010  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2011  {
2012  return __vector_bitcast<value_type>(
2013  _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2014  }
2015 #endif
2016  return __concat(__concat(__x, _Tp()),
2017  __vector_type_t<value_type, _Np * 2>());
2018  }
2019  else if constexpr (_ToN == 8 * _Np)
2020  return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2021  __vector_type_t<value_type, _Np * 4>());
2022  else if constexpr (_ToN == 16 * _Np)
2023  return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2024  __vector_type_t<value_type, _Np * 8>());
2025  else
2026  __assert_unreachable<_Tp>();
2027  }
2028  };
2029 
2030 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2031  _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2032  __zero_extend(_Tp __x)
2033  { return {__x}; }
2034 
2035 // }}}
2036 // __extract<_Np, By>{{{
2037 template <int _Offset,
2038  int _SplitBy,
2039  typename _Tp,
2040  typename _TVT = _VectorTraits<_Tp>,
2041  typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2042  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2043  __extract(_Tp __in)
2044  {
2045  using value_type = typename _TVT::value_type;
2046 #if _GLIBCXX_SIMD_X86INTRIN // {{{
2047  if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2048  {
2049  if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2050  return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2051  else if constexpr (is_floating_point_v<value_type>)
2052  return __vector_bitcast<value_type>(
2053  _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2054  else
2055  return reinterpret_cast<_R>(
2056  _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2057  _Offset));
2058  }
2059  else
2060 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
2061  {
2062 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2063  using _W = conditional_t<
2064  is_floating_point_v<value_type>, double,
2065  conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2066  static_assert(sizeof(_R) % sizeof(_W) == 0);
2067  constexpr int __return_width = sizeof(_R) / sizeof(_W);
2068  using _Up = __vector_type_t<_W, __return_width>;
2069  const auto __x = __vector_bitcast<_W>(__in);
2070 #else
2071  constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2072  using _Up = _R;
2073  const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2074  = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2075 #endif
2076  constexpr int _O = _Offset * __return_width;
2077  return __call_with_subscripts<__return_width, _O>(
2078  __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2079  return reinterpret_cast<_R>(_Up{__entries...});
2080  });
2081  }
2082  }
2083 
2084 // }}}
2085 // __lo/__hi64[z]{{{
2086 template <typename _Tp,
2087  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2088  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2089  __lo64(_Tp __x)
2090  {
2091  _R __r{};
2092  __builtin_memcpy(&__r, &__x, 8);
2093  return __r;
2094  }
2095 
2096 template <typename _Tp,
2097  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2098  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2099  __hi64(_Tp __x)
2100  {
2101  static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2102  _R __r{};
2103  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2104  return __r;
2105  }
2106 
2107 template <typename _Tp,
2108  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2109  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2110  __hi64z([[maybe_unused]] _Tp __x)
2111  {
2112  _R __r{};
2113  if constexpr (sizeof(_Tp) == 16)
2114  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2115  return __r;
2116  }
2117 
2118 // }}}
2119 // __lo/__hi128{{{
2120 template <typename _Tp>
2121  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2122  __lo128(_Tp __x)
2123  { return __extract<0, sizeof(_Tp) / 16>(__x); }
2124 
2125 template <typename _Tp>
2126  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2127  __hi128(_Tp __x)
2128  {
2129  static_assert(sizeof(__x) == 32);
2130  return __extract<1, 2>(__x);
2131  }
2132 
2133 // }}}
2134 // __lo/__hi256{{{
2135 template <typename _Tp>
2136  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2137  __lo256(_Tp __x)
2138  {
2139  static_assert(sizeof(__x) == 64);
2140  return __extract<0, 2>(__x);
2141  }
2142 
2143 template <typename _Tp>
2144  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2145  __hi256(_Tp __x)
2146  {
2147  static_assert(sizeof(__x) == 64);
2148  return __extract<1, 2>(__x);
2149  }
2150 
2151 // }}}
2152 // __auto_bitcast{{{
2153 template <typename _Tp>
2154  struct _AutoCast
2155  {
2156  static_assert(__is_vector_type_v<_Tp>);
2157 
2158  const _Tp __x;
2159 
2160  template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2161  _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2162  { return __intrin_bitcast<typename _UVT::type>(__x); }
2163  };
2164 
2165 template <typename _Tp>
2166  _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2167  __auto_bitcast(const _Tp& __x)
2168  { return {__x}; }
2169 
2170 template <typename _Tp, size_t _Np>
2171  _GLIBCXX_SIMD_INTRINSIC constexpr
2172  _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2173  __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2174  { return {__x._M_data}; }
2175 
2176 // }}}
2177 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2178 
2179 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2180 // __bool_storage_member_type{{{
2181 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2182 template <size_t _Size>
2183  struct __bool_storage_member_type
2184  {
2185  static_assert((_Size & (_Size - 1)) != 0,
2186  "This trait may only be used for non-power-of-2 sizes. "
2187  "Power-of-2 sizes must be specialized.");
2188  using type =
2189  typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2190  };
2191 
2192 template <>
2193  struct __bool_storage_member_type<1> { using type = bool; };
2194 
2195 template <>
2196  struct __bool_storage_member_type<2> { using type = __mmask8; };
2197 
2198 template <>
2199  struct __bool_storage_member_type<4> { using type = __mmask8; };
2200 
2201 template <>
2202  struct __bool_storage_member_type<8> { using type = __mmask8; };
2203 
2204 template <>
2205  struct __bool_storage_member_type<16> { using type = __mmask16; };
2206 
2207 template <>
2208  struct __bool_storage_member_type<32> { using type = __mmask32; };
2209 
2210 template <>
2211  struct __bool_storage_member_type<64> { using type = __mmask64; };
2212 #endif // _GLIBCXX_SIMD_HAVE_AVX512F
2213 
2214 // }}}
2215 // __intrinsic_type (x86){{{
2216 // the following excludes bool via __is_vectorizable
2217 #if _GLIBCXX_SIMD_HAVE_SSE
2218 template <typename _Tp, size_t _Bytes>
2219  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2220  {
2221  static_assert(!is_same_v<_Tp, long double>,
2222  "no __intrinsic_type support for long double on x86");
2223 
2224  static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2225 
2226  using type [[__gnu__::__vector_size__(_S_VBytes)]]
2227  = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2228  };
2229 #endif // _GLIBCXX_SIMD_HAVE_SSE
2230 
2231 // }}}
2232 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2233 // __intrinsic_type (ARM){{{
2234 #if _GLIBCXX_SIMD_HAVE_NEON
2235 template <>
2236  struct __intrinsic_type<float, 8, void>
2237  { using type = float32x2_t; };
2238 
2239 template <>
2240  struct __intrinsic_type<float, 16, void>
2241  { using type = float32x4_t; };
2242 
2243 template <>
2244  struct __intrinsic_type<double, 8, void>
2245  {
2246 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2247  using type = float64x1_t;
2248 #endif
2249  };
2250 
2251 template <>
2252  struct __intrinsic_type<double, 16, void>
2253  {
2254 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2255  using type = float64x2_t;
2256 #endif
2257  };
2258 
2259 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2260 template <> \
2261  struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2262  _Np * _Bits / 8, void> \
2263  { using type = int##_Bits##x##_Np##_t; }; \
2264 template <> \
2265  struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2266  _Np * _Bits / 8, void> \
2267  { using type = uint##_Bits##x##_Np##_t; }
2268 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2269 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2270 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2271 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2272 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2273 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2274 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2275 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2276 #undef _GLIBCXX_SIMD_ARM_INTRIN
2277 
2278 template <typename _Tp, size_t _Bytes>
2279  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2280  {
2281  static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2282 
2283  using _Ip = __int_for_sizeof_t<_Tp>;
2284 
2285  using _Up = conditional_t<
2286  is_floating_point_v<_Tp>, _Tp,
2287  conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2288 
2289  static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2290  "should use explicit specialization above");
2291 
2292  using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2293  };
2294 #endif // _GLIBCXX_SIMD_HAVE_NEON
2295 
2296 // }}}
2297 // __intrinsic_type (PPC){{{
2298 #ifdef __ALTIVEC__
2299 template <typename _Tp>
2300  struct __intrinsic_type_impl;
2301 
2302 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2303  template <> \
2304  struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2305 _GLIBCXX_SIMD_PPC_INTRIN(float);
2306 #ifdef __VSX__
2307 _GLIBCXX_SIMD_PPC_INTRIN(double);
2308 #endif
2309 _GLIBCXX_SIMD_PPC_INTRIN(signed char);
2310 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2311 _GLIBCXX_SIMD_PPC_INTRIN(signed short);
2312 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2313 _GLIBCXX_SIMD_PPC_INTRIN(signed int);
2314 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2315 #if defined __VSX__ || __SIZEOF_LONG__ == 4
2316 _GLIBCXX_SIMD_PPC_INTRIN(signed long);
2317 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2318 #endif
2319 #ifdef __VSX__
2320 _GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2321 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2322 #endif
2323 #undef _GLIBCXX_SIMD_PPC_INTRIN
2324 
2325 template <typename _Tp, size_t _Bytes>
2326  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2327  {
2328  static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2329 
2330  // allow _Tp == long double with -mlong-double-64
2331  static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2332  "no __intrinsic_type support for 128-bit floating point on PowerPC");
2333 
2334 #ifndef __VSX__
2335  static_assert(!(is_same_v<_Tp, double>
2336  || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2337  "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2338 #endif
2339 
2340  static constexpr auto __element_type()
2341  {
2342  if constexpr (is_floating_point_v<_Tp>)
2343  {
2344  if constexpr (_S_is_ldouble)
2345  return double {};
2346  else
2347  return _Tp {};
2348  }
2349  else if constexpr (is_signed_v<_Tp>)
2350  {
2351  if constexpr (sizeof(_Tp) == sizeof(_SChar))
2352  return _SChar {};
2353  else if constexpr (sizeof(_Tp) == sizeof(short))
2354  return short {};
2355  else if constexpr (sizeof(_Tp) == sizeof(int))
2356  return int {};
2357  else if constexpr (sizeof(_Tp) == sizeof(_LLong))
2358  return _LLong {};
2359  }
2360  else
2361  {
2362  if constexpr (sizeof(_Tp) == sizeof(_UChar))
2363  return _UChar {};
2364  else if constexpr (sizeof(_Tp) == sizeof(_UShort))
2365  return _UShort {};
2366  else if constexpr (sizeof(_Tp) == sizeof(_UInt))
2367  return _UInt {};
2368  else if constexpr (sizeof(_Tp) == sizeof(_ULLong))
2369  return _ULLong {};
2370  }
2371  }
2372 
2373  using type = typename __intrinsic_type_impl<decltype(__element_type())>::type;
2374  };
2375 #endif // __ALTIVEC__
2376 
2377 // }}}
2378 // _SimdWrapper<bool>{{{1
2379 template <size_t _Width>
2380  struct _SimdWrapper<bool, _Width,
2381  void_t<typename __bool_storage_member_type<_Width>::type>>
2382  {
2383  using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2384  using value_type = bool;
2385 
2386  static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2387 
2388  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2389  __as_full_vector() const
2390  { return _M_data; }
2391 
2392  _GLIBCXX_SIMD_INTRINSIC constexpr
2393  _SimdWrapper() = default;
2394 
2395  _GLIBCXX_SIMD_INTRINSIC constexpr
2396  _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2397 
2398  _GLIBCXX_SIMD_INTRINSIC
2399  operator const _BuiltinType&() const
2400  { return _M_data; }
2401 
2402  _GLIBCXX_SIMD_INTRINSIC
2403  operator _BuiltinType&()
2404  { return _M_data; }
2405 
2406  _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2407  __intrin() const
2408  { return _M_data; }
2409 
2410  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2411  operator[](size_t __i) const
2412  { return _M_data & (_BuiltinType(1) << __i); }
2413 
2414  template <size_t __i>
2415  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2416  operator[](_SizeConstant<__i>) const
2417  { return _M_data & (_BuiltinType(1) << __i); }
2418 
2419  _GLIBCXX_SIMD_INTRINSIC constexpr void
2420  _M_set(size_t __i, value_type __x)
2421  {
2422  if (__x)
2423  _M_data |= (_BuiltinType(1) << __i);
2424  else
2425  _M_data &= ~(_BuiltinType(1) << __i);
2426  }
2427 
2428  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2429  _M_is_constprop() const
2430  { return __builtin_constant_p(_M_data); }
2431 
2432  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2433  _M_is_constprop_none_of() const
2434  {
2435  if (__builtin_constant_p(_M_data))
2436  {
2437  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2438  constexpr _BuiltinType __active_mask
2439  = ~_BuiltinType() >> (__nbits - _Width);
2440  return (_M_data & __active_mask) == 0;
2441  }
2442  return false;
2443  }
2444 
2445  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2446  _M_is_constprop_all_of() const
2447  {
2448  if (__builtin_constant_p(_M_data))
2449  {
2450  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2451  constexpr _BuiltinType __active_mask
2452  = ~_BuiltinType() >> (__nbits - _Width);
2453  return (_M_data & __active_mask) == __active_mask;
2454  }
2455  return false;
2456  }
2457 
2458  _BuiltinType _M_data;
2459  };
2460 
2461 // _SimdWrapperBase{{{1
2462 template <bool _MustZeroInitPadding, typename _BuiltinType>
2463  struct _SimdWrapperBase;
2464 
2465 template <typename _BuiltinType>
2466  struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2467  {
2468  _GLIBCXX_SIMD_INTRINSIC constexpr
2469  _SimdWrapperBase() = default;
2470 
2471  _GLIBCXX_SIMD_INTRINSIC constexpr
2472  _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2473 
2474  _BuiltinType _M_data;
2475  };
2476 
2477 template <typename _BuiltinType>
2478  struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2479  // never become SNaN
2480  {
2481  _GLIBCXX_SIMD_INTRINSIC constexpr
2482  _SimdWrapperBase() : _M_data() {}
2483 
2484  _GLIBCXX_SIMD_INTRINSIC constexpr
2485  _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2486 
2487  _BuiltinType _M_data;
2488  };
2489 
2490 // }}}
2491 // _SimdWrapper{{{
2492 template <typename _Tp, size_t _Width>
2493  struct _SimdWrapper<
2494  _Tp, _Width,
2495  void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2496  : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2497  && sizeof(_Tp) * _Width
2498  == sizeof(__vector_type_t<_Tp, _Width>),
2499  __vector_type_t<_Tp, _Width>>
2500  {
2501  using _Base
2502  = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2503  && sizeof(_Tp) * _Width
2504  == sizeof(__vector_type_t<_Tp, _Width>),
2505  __vector_type_t<_Tp, _Width>>;
2506 
2507  static_assert(__is_vectorizable_v<_Tp>);
2508  static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2509 
2510  using _BuiltinType = __vector_type_t<_Tp, _Width>;
2511  using value_type = _Tp;
2512 
2513  static inline constexpr size_t _S_full_size
2514  = sizeof(_BuiltinType) / sizeof(value_type);
2515  static inline constexpr int _S_size = _Width;
2516  static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2517 
2518  using _Base::_M_data;
2519 
2520  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2521  __as_full_vector() const
2522  { return _M_data; }
2523 
2524  _GLIBCXX_SIMD_INTRINSIC constexpr
2525  _SimdWrapper(initializer_list<_Tp> __init)
2526  : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2527  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2528  return __init.begin()[__i.value];
2529  })) {}
2530 
2531  _GLIBCXX_SIMD_INTRINSIC constexpr
2532  _SimdWrapper() = default;
2533 
2534  _GLIBCXX_SIMD_INTRINSIC constexpr
2535  _SimdWrapper(const _SimdWrapper&) = default;
2536 
2537  _GLIBCXX_SIMD_INTRINSIC constexpr
2538  _SimdWrapper(_SimdWrapper&&) = default;
2539 
2540  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2541  operator=(const _SimdWrapper&) = default;
2542 
2543  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2544  operator=(_SimdWrapper&&) = default;
2545 
2546  template <typename _V, typename = enable_if_t<disjunction_v<
2547  is_same<_V, __vector_type_t<_Tp, _Width>>,
2548  is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2549  _GLIBCXX_SIMD_INTRINSIC constexpr
2550  _SimdWrapper(_V __x)
2551  // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2552  : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2553 
2554  template <typename... _As,
2555  typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2556  && sizeof...(_As) <= _Width)>>
2557  _GLIBCXX_SIMD_INTRINSIC constexpr
2558  operator _SimdTuple<_Tp, _As...>() const
2559  {
2560  return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2561  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2562  { return _M_data[int(__i)]; });
2563  }
2564 
2565  _GLIBCXX_SIMD_INTRINSIC constexpr
2566  operator const _BuiltinType&() const
2567  { return _M_data; }
2568 
2569  _GLIBCXX_SIMD_INTRINSIC constexpr
2570  operator _BuiltinType&()
2571  { return _M_data; }
2572 
2573  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2574  operator[](size_t __i) const
2575  { return _M_data[__i]; }
2576 
2577  template <size_t __i>
2578  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2579  operator[](_SizeConstant<__i>) const
2580  { return _M_data[__i]; }
2581 
2582  _GLIBCXX_SIMD_INTRINSIC constexpr void
2583  _M_set(size_t __i, _Tp __x)
2584  {
2585  if (__builtin_is_constant_evaluated())
2586  _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) {
2587  return __j == __i ? __x : _M_data[__j()];
2588  });
2589  else
2590  _M_data[__i] = __x;
2591  }
2592 
2593  _GLIBCXX_SIMD_INTRINSIC
2594  constexpr bool
2595  _M_is_constprop() const
2596  { return __builtin_constant_p(_M_data); }
2597 
2598  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2599  _M_is_constprop_none_of() const
2600  {
2601  if (__builtin_constant_p(_M_data))
2602  {
2603  bool __r = true;
2604  if constexpr (is_floating_point_v<_Tp>)
2605  {
2606  using _Ip = __int_for_sizeof_t<_Tp>;
2607  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2608  __execute_n_times<_Width>(
2609  [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2610  }
2611  else
2612  __execute_n_times<_Width>(
2613  [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2614  if (__builtin_constant_p(__r))
2615  return __r;
2616  }
2617  return false;
2618  }
2619 
2620  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2621  _M_is_constprop_all_of() const
2622  {
2623  if (__builtin_constant_p(_M_data))
2624  {
2625  bool __r = true;
2626  if constexpr (is_floating_point_v<_Tp>)
2627  {
2628  using _Ip = __int_for_sizeof_t<_Tp>;
2629  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2630  __execute_n_times<_Width>(
2631  [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2632  }
2633  else
2634  __execute_n_times<_Width>(
2635  [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2636  if (__builtin_constant_p(__r))
2637  return __r;
2638  }
2639  return false;
2640  }
2641  };
2642 
2643 // }}}
2644 
2645 // __vectorized_sizeof {{{
2646 template <typename _Tp>
2647  constexpr size_t
2648  __vectorized_sizeof()
2649  {
2650  if constexpr (!__is_vectorizable_v<_Tp>)
2651  return 0;
2652 
2653  if constexpr (sizeof(_Tp) <= 8)
2654  {
2655  // X86:
2656  if constexpr (__have_avx512bw)
2657  return 64;
2658  if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2659  return 64;
2660  if constexpr (__have_avx2)
2661  return 32;
2662  if constexpr (__have_avx && is_floating_point_v<_Tp>)
2663  return 32;
2664  if constexpr (__have_sse2)
2665  return 16;
2666  if constexpr (__have_sse && is_same_v<_Tp, float>)
2667  return 16;
2668  /* The following is too much trouble because of mixed MMX and x87 code.
2669  * While nothing here explicitly calls MMX instructions of registers,
2670  * they are still emitted but no EMMS cleanup is done.
2671  if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2672  return 8;
2673  */
2674 
2675  // PowerPC:
2676  if constexpr (__have_power8vec
2677  || (__have_power_vmx && (sizeof(_Tp) < 8))
2678  || (__have_power_vsx && is_floating_point_v<_Tp>) )
2679  return 16;
2680 
2681  // ARM:
2682  if constexpr (__have_neon_a64
2683  || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2684  return 16;
2685  if constexpr (__have_neon
2686  && sizeof(_Tp) < 8
2687  // Only allow fp if the user allows non-ICE559 fp (e.g.
2688  // via -ffast-math). ARMv7 NEON fp is not conforming to
2689  // IEC559.
2690  && (__support_neon_float || !is_floating_point_v<_Tp>))
2691  return 16;
2692  }
2693 
2694  return sizeof(_Tp);
2695  }
2696 
2697 // }}}
2698 namespace simd_abi {
2699 // most of simd_abi is defined in simd_detail.h
2700 template <typename _Tp>
2701  inline constexpr int max_fixed_size
2702  = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2703 
2704 // compatible {{{
2705 #if defined __x86_64__ || defined __aarch64__
2706 template <typename _Tp>
2707  using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2708 #elif defined __ARM_NEON
2709 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2710 // ABI?)
2711 template <typename _Tp>
2712  using compatible
2713  = conditional_t<(sizeof(_Tp) < 8
2714  && (__support_neon_float || !is_floating_point_v<_Tp>)),
2715  _VecBuiltin<16>, scalar>;
2716 #else
2717 template <typename>
2718  using compatible = scalar;
2719 #endif
2720 
2721 // }}}
2722 // native {{{
2723 template <typename _Tp>
2724  constexpr auto
2725  __determine_native_abi()
2726  {
2727  constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2728  if constexpr (__bytes == sizeof(_Tp))
2729  return static_cast<scalar*>(nullptr);
2730  else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2731  return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2732  else
2733  return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2734  }
2735 
2736 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2737  using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2738 
2739 // }}}
2740 // __default_abi {{{
2741 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2742 template <typename _Tp>
2743  using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2744 #else
2745 template <typename _Tp>
2746  using __default_abi = compatible<_Tp>;
2747 #endif
2748 
2749 // }}}
2750 } // namespace simd_abi
2751 
2752 // traits {{{1
2753 template <typename _Tp>
2754  struct is_simd_flag_type
2755  : false_type
2756  {};
2757 
2758 template <>
2759  struct is_simd_flag_type<element_aligned_tag>
2760  : true_type
2761  {};
2762 
2763 template <>
2764  struct is_simd_flag_type<vector_aligned_tag>
2765  : true_type
2766  {};
2767 
2768 template <size_t _Np>
2769  struct is_simd_flag_type<overaligned_tag<_Np>>
2770  : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
2771  {};
2772 
2773 template <typename _Tp>
2774  inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
2775 
2776 template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
2777  using _IsSimdFlagType = _Tp;
2778 
2779 // is_abi_tag {{{2
2780 template <typename _Tp, typename = void_t<>>
2781  struct is_abi_tag : false_type {};
2782 
2783 template <typename _Tp>
2784  struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2785  : public _Tp::_IsValidAbiTag {};
2786 
2787 template <typename _Tp>
2788  inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2789 
2790 // is_simd(_mask) {{{2
2791 template <typename _Tp>
2792  struct is_simd : public false_type {};
2793 
2794 template <typename _Tp>
2795  inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2796 
2797 template <typename _Tp>
2798  struct is_simd_mask : public false_type {};
2799 
2800 template <typename _Tp>
2801 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2802 
2803 // simd_size {{{2
2804 template <typename _Tp, typename _Abi, typename = void>
2805  struct __simd_size_impl {};
2806 
2807 template <typename _Tp, typename _Abi>
2808  struct __simd_size_impl<
2809  _Tp, _Abi,
2810  enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2811  : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2812 
2813 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2814  struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2815 
2816 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2817  inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2818 
2819 // simd_abi::deduce {{{2
2820 template <typename _Tp, size_t _Np, typename = void>
2821  struct __deduce_impl;
2822 
2823 namespace simd_abi {
2824 /**
2825  * @tparam _Tp The requested `value_type` for the elements.
2826  * @tparam _Np The requested number of elements.
2827  * @tparam _Abis This parameter is ignored, since this implementation cannot
2828  * make any use of it. Either __a good native ABI is matched and used as `type`
2829  * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2830  * the best matching native ABIs.
2831  */
2832 template <typename _Tp, size_t _Np, typename...>
2833  struct deduce : __deduce_impl<_Tp, _Np> {};
2834 
2835 template <typename _Tp, size_t _Np, typename... _Abis>
2836  using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2837 } // namespace simd_abi
2838 
2839 // }}}2
2840 // rebind_simd {{{2
2841 template <typename _Tp, typename _V, typename = void>
2842  struct rebind_simd;
2843 
2844 template <typename _Tp, typename _Up, typename _Abi>
2845  struct rebind_simd<_Tp, simd<_Up, _Abi>,
2846  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2847  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2848 
2849 template <typename _Tp, typename _Up, typename _Abi>
2850  struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
2851  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2852  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2853 
2854 template <typename _Tp, typename _V>
2855  using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2856 
2857 // resize_simd {{{2
2858 template <int _Np, typename _V, typename = void>
2859  struct resize_simd;
2860 
2861 template <int _Np, typename _Tp, typename _Abi>
2862  struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2863  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2864 
2865 template <int _Np, typename _Tp, typename _Abi>
2866  struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2867  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2868 
2869 template <int _Np, typename _V>
2870  using resize_simd_t = typename resize_simd<_Np, _V>::type;
2871 
2872 // }}}2
2873 // memory_alignment {{{2
2874 template <typename _Tp, typename _Up = typename _Tp::value_type>
2875  struct memory_alignment
2876  : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2877 
2878 template <typename _Tp, typename _Up = typename _Tp::value_type>
2879  inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2880 
2881 // class template simd [simd] {{{1
2882 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2883  class simd;
2884 
2885 template <typename _Tp, typename _Abi>
2886  struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2887 
2888 template <typename _Tp>
2889  using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2890 
2891 template <typename _Tp, int _Np>
2892  using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2893 
2894 template <typename _Tp, size_t _Np>
2895  using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2896 
2897 // class template simd_mask [simd_mask] {{{1
2898 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2899  class simd_mask;
2900 
2901 template <typename _Tp, typename _Abi>
2902  struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2903 
2904 template <typename _Tp>
2905  using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2906 
2907 template <typename _Tp, int _Np>
2908  using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2909 
2910 template <typename _Tp, size_t _Np>
2911  using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2912 
2913 // casts [simd.casts] {{{1
2914 // static_simd_cast {{{2
2915 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void>
2916  struct __static_simd_cast_return_type;
2917 
2918 template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2919  struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
2920  : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2921 
2922 template <typename _Tp, typename _Up, typename _Ap>
2923  struct __static_simd_cast_return_type<
2924  _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2925  { using type = _Tp; };
2926 
2927 template <typename _Tp, typename _Ap>
2928  struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2929 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2930  enable_if_t<__is_vectorizable_v<_Tp>>
2931 #else
2932  void
2933 #endif
2934  >
2935  { using type = simd<_Tp, _Ap>; };
2936 
2937 template <typename _Tp, typename = void>
2938  struct __safe_make_signed { using type = _Tp;};
2939 
2940 template <typename _Tp>
2941  struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2942  {
2943  // the extra make_unsigned_t is because of PR85951
2944  using type = make_signed_t<make_unsigned_t<_Tp>>;
2945  };
2946 
2947 template <typename _Tp>
2948  using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2949 
2950 template <typename _Tp, typename _Up, typename _Ap>
2951  struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2952 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2953  enable_if_t<__is_vectorizable_v<_Tp>>
2954 #else
2955  void
2956 #endif
2957  >
2958  {
2959  using type = conditional_t<
2960  (is_integral_v<_Up> && is_integral_v<_Tp> &&
2961 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2962  is_signed_v<_Up> != is_signed_v<_Tp> &&
2963 #endif
2964  is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2965  simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2966  };
2967 
2968 template <typename _Tp, typename _Up, typename _Ap,
2969  typename _R
2970  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2971  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2972  static_simd_cast(const simd<_Up, _Ap>& __x)
2973  {
2974  if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2975  return __x;
2976  else
2977  {
2978  _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2979  __c;
2980  return _R(__private_init, __c(__data(__x)));
2981  }
2982  }
2983 
2984 namespace __proposed {
2985 template <typename _Tp, typename _Up, typename _Ap,
2986  typename _R
2987  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2988  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2989  static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2990  {
2991  using _RM = typename _R::mask_type;
2992  return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2993  typename _RM::simd_type::value_type>(__x)};
2994  }
2995 } // namespace __proposed
2996 
2997 // simd_cast {{{2
2998 template <typename _Tp, typename _Up, typename _Ap,
2999  typename _To = __value_type_or_identity_t<_Tp>>
3000  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3001  simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3002  -> decltype(static_simd_cast<_Tp>(__x))
3003  { return static_simd_cast<_Tp>(__x); }
3004 
3005 namespace __proposed {
3006 template <typename _Tp, typename _Up, typename _Ap,
3007  typename _To = __value_type_or_identity_t<_Tp>>
3008  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3009  simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3010  -> decltype(static_simd_cast<_Tp>(__x))
3011  { return static_simd_cast<_Tp>(__x); }
3012 } // namespace __proposed
3013 
3014 // }}}2
3015 // resizing_simd_cast {{{
3016 namespace __proposed {
3017 /* Proposed spec:
3018 
3019 template <class T, class U, class Abi>
3020 T resizing_simd_cast(const simd<U, Abi>& x)
3021 
3022 p1 Constraints:
3023  - is_simd_v<T> is true and
3024  - T::value_type is the same type as U
3025 
3026 p2 Returns:
3027  A simd object with the i^th element initialized to x[i] for all i in the
3028  range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3029  than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3030 
3031 template <class T, class U, class Abi>
3032 T resizing_simd_cast(const simd_mask<U, Abi>& x)
3033 
3034 p1 Constraints: is_simd_mask_v<T> is true
3035 
3036 p2 Returns:
3037  A simd_mask object with the i^th element initialized to x[i] for all i in
3038 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3039  than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3040 
3041  */
3042 
3043 template <typename _Tp, typename _Up, typename _Ap>
3044  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3045  conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3046  resizing_simd_cast(const simd<_Up, _Ap>& __x)
3047  {
3048  if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3049  return __x;
3050  else if (__builtin_is_constant_evaluated())
3051  return _Tp([&](auto __i) constexpr {
3052  return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3053  });
3054  else if constexpr (simd_size_v<_Up, _Ap> == 1)
3055  {
3056  _Tp __r{};
3057  __r[0] = __x[0];
3058  return __r;
3059  }
3060  else if constexpr (_Tp::size() == 1)
3061  return __x[0];
3062  else if constexpr (sizeof(_Tp) == sizeof(__x)
3063  && !__is_fixed_size_abi_v<_Ap>)
3064  return {__private_init,
3065  __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3066  _Ap::_S_masked(__data(__x))._M_data)};
3067  else
3068  {
3069  _Tp __r{};
3070  __builtin_memcpy(&__data(__r), &__data(__x),
3071  sizeof(_Up)
3072  * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3073  return __r;
3074  }
3075  }
3076 
3077 template <typename _Tp, typename _Up, typename _Ap>
3078  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3079  enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3080  resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3081  {
3082  return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3083  typename _Tp::simd_type::value_type>(__x)};
3084  }
3085 } // namespace __proposed
3086 
3087 // }}}
3088 // to_fixed_size {{{2
3089 template <typename _Tp, int _Np>
3090  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3091  to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3092  { return __x; }
3093 
3094 template <typename _Tp, int _Np>
3095  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3096  to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3097  { return __x; }
3098 
3099 template <typename _Tp, typename _Ap>
3100  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3101  to_fixed_size(const simd<_Tp, _Ap>& __x)
3102  {
3103  using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3104  return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3105  }
3106 
3107 template <typename _Tp, typename _Ap>
3108  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3109  to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3110  {
3111  return {__private_init,
3112  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3113  }
3114 
3115 // to_native {{{2
3116 template <typename _Tp, int _Np>
3117  _GLIBCXX_SIMD_INTRINSIC
3118  enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3119  to_native(const fixed_size_simd<_Tp, _Np>& __x)
3120  {
3121  alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3122  __x.copy_to(__mem, vector_aligned);
3123  return {__mem, vector_aligned};
3124  }
3125 
3126 template <typename _Tp, int _Np>
3127  _GLIBCXX_SIMD_INTRINSIC
3128  enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3129  to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3130  {
3131  return native_simd_mask<_Tp>(
3132  __private_init,
3133  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3134  }
3135 
3136 // to_compatible {{{2
3137 template <typename _Tp, int _Np>
3138  _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3139  to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3140  {
3141  alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3142  __x.copy_to(__mem, vector_aligned);
3143  return {__mem, vector_aligned};
3144  }
3145 
3146 template <typename _Tp, int _Np>
3147  _GLIBCXX_SIMD_INTRINSIC
3148  enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3149  to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3150  {
3151  return simd_mask<_Tp>(
3152  __private_init,
3153  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3154  }
3155 
3156 // masked assignment [simd_mask.where] {{{1
3157 
3158 // where_expression {{{1
3159 // const_where_expression<M, T> {{{2
3160 template <typename _M, typename _Tp>
3161  class const_where_expression
3162  {
3163  using _V = _Tp;
3164  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3165 
3166  struct _Wrapper { using value_type = _V; };
3167 
3168  protected:
3169  using _Impl = typename _V::_Impl;
3170 
3171  using value_type =
3172  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3173 
3174  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3175  __get_mask(const const_where_expression& __x)
3176  { return __x._M_k; }
3177 
3178  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3179  __get_lvalue(const const_where_expression& __x)
3180  { return __x._M_value; }
3181 
3182  const _M& _M_k;
3183  _Tp& _M_value;
3184 
3185  public:
3186  const_where_expression(const const_where_expression&) = delete;
3187 
3188  const_where_expression& operator=(const const_where_expression&) = delete;
3189 
3190  _GLIBCXX_SIMD_INTRINSIC constexpr
3191  const_where_expression(const _M& __kk, const _Tp& dd)
3192  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3193 
3194  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3195  operator-() const&&
3196  {
3197  return {__private_init,
3198  _Impl::template _S_masked_unary<negate>(__data(_M_k),
3199  __data(_M_value))};
3200  }
3201 
3202  template <typename _Up, typename _Flags>
3203  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3204  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3205  {
3206  return {__private_init,
3207  _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3208  _Flags::template _S_apply<_V>(__mem))};
3209  }
3210 
3211  template <typename _Up, typename _Flags>
3212  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3213  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3214  {
3215  _Impl::_S_masked_store(__data(_M_value),
3216  _Flags::template _S_apply<_V>(__mem),
3217  __data(_M_k));
3218  }
3219  };
3220 
3221 // const_where_expression<bool, T> {{{2
3222 template <typename _Tp>
3223  class const_where_expression<bool, _Tp>
3224  {
3225  using _M = bool;
3226  using _V = _Tp;
3227 
3228  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3229 
3230  struct _Wrapper { using value_type = _V; };
3231 
3232  protected:
3233  using value_type
3234  = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3235 
3236  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3237  __get_mask(const const_where_expression& __x)
3238  { return __x._M_k; }
3239 
3240  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3241  __get_lvalue(const const_where_expression& __x)
3242  { return __x._M_value; }
3243 
3244  const bool _M_k;
3245  _Tp& _M_value;
3246 
3247  public:
3248  const_where_expression(const const_where_expression&) = delete;
3249  const_where_expression& operator=(const const_where_expression&) = delete;
3250 
3251  _GLIBCXX_SIMD_INTRINSIC constexpr
3252  const_where_expression(const bool __kk, const _Tp& dd)
3253  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3254 
3255  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3256  operator-() const&&
3257  { return _M_k ? -_M_value : _M_value; }
3258 
3259  template <typename _Up, typename _Flags>
3260  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3261  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3262  { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3263 
3264  template <typename _Up, typename _Flags>
3265  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3266  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3267  {
3268  if (_M_k)
3269  __mem[0] = _M_value;
3270  }
3271  };
3272 
3273 // where_expression<M, T> {{{2
3274 template <typename _M, typename _Tp>
3275  class where_expression : public const_where_expression<_M, _Tp>
3276  {
3277  using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3278 
3279  static_assert(!is_const<_Tp>::value,
3280  "where_expression may only be instantiated with __a non-const "
3281  "_Tp parameter");
3282 
3283  using typename const_where_expression<_M, _Tp>::value_type;
3284  using const_where_expression<_M, _Tp>::_M_k;
3285  using const_where_expression<_M, _Tp>::_M_value;
3286 
3287  static_assert(
3288  is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3289  static_assert(_M::size() == _Tp::size(), "");
3290 
3291  _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp&
3292  __get_lvalue(where_expression& __x)
3293  { return __x._M_value; }
3294 
3295  public:
3296  where_expression(const where_expression&) = delete;
3297  where_expression& operator=(const where_expression&) = delete;
3298 
3299  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3300  where_expression(const _M& __kk, _Tp& dd)
3301  : const_where_expression<_M, _Tp>(__kk, dd) {}
3302 
3303  template <typename _Up>
3304  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3305  operator=(_Up&& __x) &&
3306  {
3307  _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3308  __to_value_type_or_member_type<_Tp>(
3309  static_cast<_Up&&>(__x)));
3310  }
3311 
3312 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3313  template <typename _Up> \
3314  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3315  operator __op##=(_Up&& __x)&& \
3316  { \
3317  _Impl::template _S_masked_cassign( \
3318  __data(_M_k), __data(_M_value), \
3319  __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3320  [](auto __impl, auto __lhs, auto __rhs) \
3321  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3322  { return __impl.__name(__lhs, __rhs); }); \
3323  } \
3324  static_assert(true)
3325  _GLIBCXX_SIMD_OP_(+, _S_plus);
3326  _GLIBCXX_SIMD_OP_(-, _S_minus);
3327  _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3328  _GLIBCXX_SIMD_OP_(/, _S_divides);
3329  _GLIBCXX_SIMD_OP_(%, _S_modulus);
3330  _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3331  _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3332  _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3333  _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3334  _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3335 #undef _GLIBCXX_SIMD_OP_
3336 
3337  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3338  operator++() &&
3339  {
3340  __data(_M_value)
3341  = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3342  }
3343 
3344  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3345  operator++(int) &&
3346  {
3347  __data(_M_value)
3348  = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3349  }
3350 
3351  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3352  operator--() &&
3353  {
3354  __data(_M_value)
3355  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3356  }
3357 
3358  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3359  operator--(int) &&
3360  {
3361  __data(_M_value)
3362  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3363  }
3364 
3365  // intentionally hides const_where_expression::copy_from
3366  template <typename _Up, typename _Flags>
3367  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3368  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3369  {
3370  __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3371  _Flags::template _S_apply<_Tp>(__mem));
3372  }
3373  };
3374 
3375 // where_expression<bool, T> {{{2
3376 template <typename _Tp>
3377  class where_expression<bool, _Tp>
3378  : public const_where_expression<bool, _Tp>
3379  {
3380  using _M = bool;
3381  using typename const_where_expression<_M, _Tp>::value_type;
3382  using const_where_expression<_M, _Tp>::_M_k;
3383  using const_where_expression<_M, _Tp>::_M_value;
3384 
3385  public:
3386  where_expression(const where_expression&) = delete;
3387  where_expression& operator=(const where_expression&) = delete;
3388 
3389  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3390  where_expression(const _M& __kk, _Tp& dd)
3391  : const_where_expression<_M, _Tp>(__kk, dd) {}
3392 
3393 #define _GLIBCXX_SIMD_OP_(__op) \
3394  template <typename _Up> \
3395  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3396  operator __op(_Up&& __x)&& \
3397  { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3398 
3399  _GLIBCXX_SIMD_OP_(=)
3400  _GLIBCXX_SIMD_OP_(+=)
3401  _GLIBCXX_SIMD_OP_(-=)
3402  _GLIBCXX_SIMD_OP_(*=)
3403  _GLIBCXX_SIMD_OP_(/=)
3404  _GLIBCXX_SIMD_OP_(%=)
3405  _GLIBCXX_SIMD_OP_(&=)
3406  _GLIBCXX_SIMD_OP_(|=)
3407  _GLIBCXX_SIMD_OP_(^=)
3408  _GLIBCXX_SIMD_OP_(<<=)
3409  _GLIBCXX_SIMD_OP_(>>=)
3410  #undef _GLIBCXX_SIMD_OP_
3411 
3412  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3413  operator++() &&
3414  { if (_M_k) ++_M_value; }
3415 
3416  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3417  operator++(int) &&
3418  { if (_M_k) ++_M_value; }
3419 
3420  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3421  operator--() &&
3422  { if (_M_k) --_M_value; }
3423 
3424  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3425  operator--(int) &&
3426  { if (_M_k) --_M_value; }
3427 
3428  // intentionally hides const_where_expression::copy_from
3429  template <typename _Up, typename _Flags>
3430  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3431  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3432  { if (_M_k) _M_value = __mem[0]; }
3433  };
3434 
3435 // where {{{1
3436 template <typename _Tp, typename _Ap>
3437  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3438  where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3439  where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3440  { return {__k, __value}; }
3441 
3442 template <typename _Tp, typename _Ap>
3443  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3444  const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3445  where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value)
3446  { return {__k, __value}; }
3447 
3448 template <typename _Tp, typename _Ap>
3449  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3450  where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3451  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3452  { return {__k, __value}; }
3453 
3454 template <typename _Tp, typename _Ap>
3455  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3456  const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3457  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value)
3458  { return {__k, __value}; }
3459 
3460 template <typename _Tp>
3461  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3462  where(_ExactBool __k, _Tp& __value)
3463  { return {__k, __value}; }
3464 
3465 template <typename _Tp>
3466  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3467  where(_ExactBool __k, const _Tp& __value)
3468  { return {__k, __value}; }
3469 
3470 template <typename _Tp, typename _Ap>
3471  _GLIBCXX_SIMD_CONSTEXPR void
3472  where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3473 
3474 template <typename _Tp, typename _Ap>
3475  _GLIBCXX_SIMD_CONSTEXPR void
3476  where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3477 
3478 // proposed mask iterations {{{1
3479 namespace __proposed {
3480 template <size_t _Np>
3481  class where_range
3482  {
3483  const bitset<_Np> __bits;
3484 
3485  public:
3486  where_range(bitset<_Np> __b) : __bits(__b) {}
3487 
3488  class iterator
3489  {
3490  size_t __mask;
3491  size_t __bit;
3492 
3493  _GLIBCXX_SIMD_INTRINSIC void
3494  __next_bit()
3495  { __bit = __builtin_ctzl(__mask); }
3496 
3497  _GLIBCXX_SIMD_INTRINSIC void
3498  __reset_lsb()
3499  {
3500  // 01100100 - 1 = 01100011
3501  __mask &= (__mask - 1);
3502  // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3503  }
3504 
3505  public:
3506  iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3507  iterator(const iterator&) = default;
3508  iterator(iterator&&) = default;
3509 
3510  _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3511  operator->() const
3512  { return __bit; }
3513 
3514  _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3515  operator*() const
3516  { return __bit; }
3517 
3518  _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3519  operator++()
3520  {
3521  __reset_lsb();
3522  __next_bit();
3523  return *this;
3524  }
3525 
3526  _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3527  operator++(int)
3528  {
3529  iterator __tmp = *this;
3530  __reset_lsb();
3531  __next_bit();
3532  return __tmp;
3533  }
3534 
3535  _GLIBCXX_SIMD_ALWAYS_INLINE bool
3536  operator==(const iterator& __rhs) const
3537  { return __mask == __rhs.__mask; }
3538 
3539  _GLIBCXX_SIMD_ALWAYS_INLINE bool
3540  operator!=(const iterator& __rhs) const
3541  { return __mask != __rhs.__mask; }
3542  };
3543 
3544  iterator
3545  begin() const
3546  { return __bits.to_ullong(); }
3547 
3548  iterator
3549  end() const
3550  { return 0; }
3551  };
3552 
3553 template <typename _Tp, typename _Ap>
3554  where_range<simd_size_v<_Tp, _Ap>>
3555  where(const simd_mask<_Tp, _Ap>& __k)
3556  { return __k.__to_bitset(); }
3557 
3558 } // namespace __proposed
3559 
3560 // }}}1
3561 // reductions [simd.reductions] {{{1
3562 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3563  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3564  reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3565  { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3566 
3567 template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3568  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3569  reduce(const const_where_expression<_M, _V>& __x,
3570  typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3571  {
3572  if (__builtin_expect(none_of(__get_mask(__x)), false))
3573  return __identity_element;
3574 
3575  _V __tmp = __identity_element;
3576  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3577  __data(__get_lvalue(__x)));
3578  return reduce(__tmp, __binary_op);
3579  }
3580 
3581 template <typename _M, typename _V>
3582  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3583  reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3584  { return reduce(__x, 0, __binary_op); }
3585 
3586 template <typename _M, typename _V>
3587  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3588  reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3589  { return reduce(__x, 1, __binary_op); }
3590 
3591 template <typename _M, typename _V>
3592  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3593  reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3594  { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3595 
3596 template <typename _M, typename _V>
3597  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3598  reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3599  { return reduce(__x, 0, __binary_op); }
3600 
3601 template <typename _M, typename _V>
3602  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3603  reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3604  { return reduce(__x, 0, __binary_op); }
3605 
3606 template <typename _Tp, typename _Abi>
3607  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3608  hmin(const simd<_Tp, _Abi>& __v) noexcept
3609  { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3610 
3611 template <typename _Tp, typename _Abi>
3612  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3613  hmax(const simd<_Tp, _Abi>& __v) noexcept
3614  { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3615 
3616 template <typename _M, typename _V>
3617  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3618  typename _V::value_type
3619  hmin(const const_where_expression<_M, _V>& __x) noexcept
3620  {
3621  using _Tp = typename _V::value_type;
3622  constexpr _Tp __id_elem =
3623 #ifdef __FINITE_MATH_ONLY__
3624  __finite_max_v<_Tp>;
3625 #else
3626  __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3627 #endif
3628  _V __tmp = __id_elem;
3629  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3630  __data(__get_lvalue(__x)));
3631  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3632  }
3633 
3634 template <typename _M, typename _V>
3635  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3636  typename _V::value_type
3637  hmax(const const_where_expression<_M, _V>& __x) noexcept
3638  {
3639  using _Tp = typename _V::value_type;
3640  constexpr _Tp __id_elem =
3641 #ifdef __FINITE_MATH_ONLY__
3642  __finite_min_v<_Tp>;
3643 #else
3644  [] {
3645  if constexpr (__value_exists_v<__infinity, _Tp>)
3646  return -__infinity_v<_Tp>;
3647  else
3648  return __finite_min_v<_Tp>;
3649  }();
3650 #endif
3651  _V __tmp = __id_elem;
3652  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3653  __data(__get_lvalue(__x)));
3654  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3655  }
3656 
3657 // }}}1
3658 // algorithms [simd.alg] {{{
3659 template <typename _Tp, typename _Ap>
3660  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3661  min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3662  { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3663 
3664 template <typename _Tp, typename _Ap>
3665  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3666  max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3667  { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3668 
3669 template <typename _Tp, typename _Ap>
3670  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3671  pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3672  minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3673  {
3674  const auto pair_of_members
3675  = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3676  return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3677  simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3678  }
3679 
3680 template <typename _Tp, typename _Ap>
3681  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3682  clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi)
3683  {
3684  using _Impl = typename _Ap::_SimdImpl;
3685  return {__private_init,
3686  _Impl::_S_min(__data(__hi),
3687  _Impl::_S_max(__data(__lo), __data(__v)))};
3688  }
3689 
3690 // }}}
3691 
3692 template <size_t... _Sizes, typename _Tp, typename _Ap,
3693  typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3694  inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3695  split(const simd<_Tp, _Ap>&);
3696 
3697 // __extract_part {{{
3698 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3699  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr
3700  _SimdWrapper<_Tp, _Np / _Total * _Combine>
3701  __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3702 
3703 template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As>
3704  _GLIBCXX_SIMD_INTRINSIC constexpr auto
3705  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3706 
3707 // }}}
3708 // _SizeList {{{
3709 template <size_t _V0, size_t... _Values>
3710  struct _SizeList
3711  {
3712  template <size_t _I>
3713  static constexpr size_t
3714  _S_at(_SizeConstant<_I> = {})
3715  {
3716  if constexpr (_I == 0)
3717  return _V0;
3718  else
3719  return _SizeList<_Values...>::template _S_at<_I - 1>();
3720  }
3721 
3722  template <size_t _I>
3723  static constexpr auto
3724  _S_before(_SizeConstant<_I> = {})
3725  {
3726  if constexpr (_I == 0)
3727  return _SizeConstant<0>();
3728  else
3729  return _SizeConstant<
3730  _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3731  }
3732 
3733  template <size_t _Np>
3734  static constexpr auto
3735  _S_pop_front(_SizeConstant<_Np> = {})
3736  {
3737  if constexpr (_Np == 0)
3738  return _SizeList();
3739  else
3740  return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3741  }
3742  };
3743 
3744 // }}}
3745 // __extract_center {{{
3746 template <typename _Tp, size_t _Np>
3747  _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3748  __extract_center(_SimdWrapper<_Tp, _Np> __x)
3749  {
3750  static_assert(_Np >= 4);
3751  static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3752 #if _GLIBCXX_SIMD_X86INTRIN // {{{
3753  if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3754  {
3755  const auto __intrin = __to_intrin(__x);
3756  if constexpr (is_integral_v<_Tp>)
3757  return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3758  _mm512_shuffle_i32x4(__intrin, __intrin,
3759  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3760  else if constexpr (sizeof(_Tp) == 4)
3761  return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3762  _mm512_shuffle_f32x4(__intrin, __intrin,
3763  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3764  else if constexpr (sizeof(_Tp) == 8)
3765  return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3766  _mm512_shuffle_f64x2(__intrin, __intrin,
3767  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3768  else
3769  __assert_unreachable<_Tp>();
3770  }
3771  else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3772  return __vector_bitcast<_Tp>(
3773  _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3774  __hi128(__vector_bitcast<double>(__x)), 1));
3775  else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3776  return __vector_bitcast<_Tp>(
3777  _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3778  __lo128(__vector_bitcast<_LLong>(__x)),
3779  sizeof(_Tp) * _Np / 4));
3780  else
3781 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
3782  {
3783  __vector_type_t<_Tp, _Np / 2> __r;
3784  __builtin_memcpy(&__r,
3785  reinterpret_cast<const char*>(&__x)
3786  + sizeof(_Tp) * _Np / 4,
3787  sizeof(_Tp) * _Np / 2);
3788  return __r;
3789  }
3790  }
3791 
3792 template <typename _Tp, typename _A0, typename... _As>
3793  _GLIBCXX_SIMD_INTRINSIC
3794  _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3795  __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3796  {
3797  if constexpr (sizeof...(_As) == 0)
3798  return __extract_center(__x.first);
3799  else
3800  return __extract_part<1, 4, 2>(__x);
3801  }
3802 
3803 // }}}
3804 // __split_wrapper {{{
3805 template <size_t... _Sizes, typename _Tp, typename... _As>
3806  auto
3807  __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3808  {
3809  return split<_Sizes...>(
3810  fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3811  __x));
3812  }
3813 
3814 // }}}
3815 
3816 // split<simd>(simd) {{{
3817 template <typename _V, typename _Ap,
3818  size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3819  enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
3820  && is_simd_v<_V>, array<_V, _Parts>>
3821  split(const simd<typename _V::value_type, _Ap>& __x)
3822  {
3823  using _Tp = typename _V::value_type;
3824  if constexpr (_Parts == 1)
3825  {
3826  return {simd_cast<_V>(__x)};
3827  }
3828  else if (__x._M_is_constprop())
3829  {
3830  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3831  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3832  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3833  { return __x[__i * _V::size() + __j]; });
3834  });
3835  }
3836  else if constexpr (
3837  __is_fixed_size_abi_v<_Ap>
3838  && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3839  || (__is_fixed_size_abi_v<typename _V::abi_type>
3840  && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3841  )))
3842  {
3843  // fixed_size -> fixed_size (w/o padding) or scalar
3844 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3845  const __may_alias<_Tp>* const __element_ptr
3846  = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3847  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3848  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3849  { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
3850 #else
3851  const auto& __xx = __data(__x);
3852  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3853  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3854  [[maybe_unused]] constexpr size_t __offset
3855  = decltype(__i)::value * _V::size();
3856  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3857  constexpr _SizeConstant<__j + __offset> __k;
3858  return __xx[__k];
3859  });
3860  });
3861 #endif
3862  }
3863  else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3864  {
3865  // normally memcpy should work here as well
3866  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3867  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3868  }
3869  else
3870  {
3871  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3872  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3873  if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3874  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3875  return __x[__i * _V::size() + __j];
3876  });
3877  else
3878  return _V(__private_init,
3879  __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
3880  });
3881  }
3882  }
3883 
3884 // }}}
3885 // split<simd_mask>(simd_mask) {{{
3886 template <typename _V, typename _Ap,
3887  size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3888  enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3889  _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3890  split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3891  {
3892  if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3893  return {__x};
3894  else if constexpr (_Parts == 1)
3895  return {__proposed::static_simd_cast<_V>(__x)};
3896  else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3897  && __is_avx_abi<_Ap>())
3898  return {_V(__private_init, __lo128(__data(__x))),
3899  _V(__private_init, __hi128(__data(__x)))};
3900  else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3901  {
3902  const bitset __bits = __x.__to_bitset();
3903  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3904  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3905  constexpr size_t __offset = __i * _V::size();
3906  return _V(__bitset_init, (__bits >> __offset).to_ullong());
3907  });
3908  }
3909  else
3910  {
3911  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3912  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3913  constexpr size_t __offset = __i * _V::size();
3914  return _V(__private_init,
3915  [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3916  return __x[__j + __offset];
3917  });
3918  });
3919  }
3920  }
3921 
3922 // }}}
3923 // split<_Sizes...>(simd) {{{
3924 template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3925  _GLIBCXX_SIMD_ALWAYS_INLINE
3926  tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3927  split(const simd<_Tp, _Ap>& __x)
3928  {
3929  using _SL = _SizeList<_Sizes...>;
3930  using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3931  constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3932  constexpr size_t _N0 = _SL::template _S_at<0>();
3933  using _V = __deduced_simd<_Tp, _N0>;
3934 
3935  if (__x._M_is_constprop())
3936  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
3937  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3938  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3939  constexpr size_t __offset = _SL::_S_before(__i);
3940  return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3941  return __x[__offset + __j];
3942  });
3943  });
3944  else if constexpr (_Np == _N0)
3945  {
3946  static_assert(sizeof...(_Sizes) == 1);
3947  return {simd_cast<_V>(__x)};
3948  }
3949  else if constexpr // split from fixed_size, such that __x::first.size == _N0
3950  (__is_fixed_size_abi_v<
3951  _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3952  {
3953  static_assert(
3954  !__is_fixed_size_abi_v<typename _V::abi_type>,
3955  "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3956  "fixed_size_simd "
3957  "when deduced?");
3958  // extract first and recurse (__split_wrapper is needed to deduce a new
3959  // _Sizes pack)
3960  return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3961  __split_wrapper(_SL::template _S_pop_front<1>(),
3962  __data(__x).second));
3963  }
3964  else if constexpr ((!is_same_v<simd_abi::scalar,
3965  simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3966  && (!__is_fixed_size_abi_v<
3967  simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3968  {
3969  if constexpr (((_Sizes * 2 == _Np) && ...))
3970  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3971  {__private_init, __extract_part<1, 2>(__data(__x))}};
3972  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3973  _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3974  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3975  {__private_init, __extract_part<1, 3>(__data(__x))},
3976  {__private_init, __extract_part<2, 3>(__data(__x))}};
3977  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3978  _SizeList<2 * _Np / 3, _Np / 3>>)
3979  return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3980  {__private_init, __extract_part<2, 3>(__data(__x))}};
3981  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3982  _SizeList<_Np / 3, 2 * _Np / 3>>)
3983  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3984  {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3985  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3986  _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3987  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3988  {__private_init, __extract_part<2, 4>(__data(__x))},
3989  {__private_init, __extract_part<3, 4>(__data(__x))}};
3990  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3991  _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3992  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3993  {__private_init, __extract_part<1, 4>(__data(__x))},
3994  {__private_init, __extract_part<1, 2>(__data(__x))}};
3995  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3996  _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3997  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3998  {__private_init, __extract_center(__data(__x))},
3999  {__private_init, __extract_part<3, 4>(__data(__x))}};
4000  else if constexpr (((_Sizes * 4 == _Np) && ...))
4001  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4002  {__private_init, __extract_part<1, 4>(__data(__x))},
4003  {__private_init, __extract_part<2, 4>(__data(__x))},
4004  {__private_init, __extract_part<3, 4>(__data(__x))}};
4005  // else fall through
4006  }
4007 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4008  const __may_alias<_Tp>* const __element_ptr
4009  = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4010  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4011  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4012  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4013  constexpr size_t __offset = _SL::_S_before(__i);
4014  constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4015  constexpr size_t __a
4016  = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4017  constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4018  constexpr size_t __alignment = __b == 0 ? __a : __b;
4019  return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4020  });
4021 #else
4022  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4023  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4024  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4025  const auto& __xx = __data(__x);
4026  using _Offset = decltype(_SL::_S_before(__i));
4027  return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4028  constexpr _SizeConstant<_Offset::value + __j> __k;
4029  return __xx[__k];
4030  });
4031  });
4032 #endif
4033  }
4034 
4035 // }}}
4036 
4037 // __subscript_in_pack {{{
4038 template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4039  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4040  __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4041  {
4042  if constexpr (_I < simd_size_v<_Tp, _Ap>)
4043  return __x[_I];
4044  else
4045  return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4046  }
4047 
4048 // }}}
4049 // __store_pack_of_simd {{{
4050 template <typename _Tp, typename _A0, typename... _As>
4051  _GLIBCXX_SIMD_INTRINSIC void
4052  __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
4053  {
4054  constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4055  __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4056  if constexpr (sizeof...(__xs) > 0)
4057  __store_pack_of_simd(__mem + __n_bytes, __xs...);
4058  }
4059 
4060 // }}}
4061 // concat(simd...) {{{
4062 template <typename _Tp, typename... _As>
4063  inline _GLIBCXX_SIMD_CONSTEXPR
4064  simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4065  concat(const simd<_Tp, _As>&... __xs)
4066  {
4067  using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4068  if constexpr (sizeof...(__xs) == 1)
4069  return simd_cast<_Rp>(__xs...);
4070  else if ((... && __xs._M_is_constprop()))
4071  return simd<_Tp,
4072  simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4073  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4074  { return __subscript_in_pack<__i>(__xs...); });
4075  else
4076  {
4077  _Rp __r{};
4078  __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4079  return __r;
4080  }
4081  }
4082 
4083 // }}}
4084 // concat(array<simd>) {{{
4085 template <typename _Tp, typename _Abi, size_t _Np>
4086  _GLIBCXX_SIMD_ALWAYS_INLINE
4087  _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4088  concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4089  {
4090  return __call_with_subscripts<_Np>(
4091  __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4092  return concat(__xs...);
4093  });
4094  }
4095 
4096 // }}}
4097 
4098 /// @cond undocumented
4099 // _SmartReference {{{
4100 template <typename _Up, typename _Accessor = _Up,
4101  typename _ValueType = typename _Up::value_type>
4102  class _SmartReference
4103  {
4104  friend _Accessor;
4105  int _M_index;
4106  _Up& _M_obj;
4107 
4108  _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType
4109  _M_read() const noexcept
4110  {
4111  if constexpr (is_arithmetic_v<_Up>)
4112  return _M_obj;
4113  else
4114  return _M_obj[_M_index];
4115  }
4116 
4117  template <typename _Tp>
4118  _GLIBCXX_SIMD_INTRINSIC constexpr void
4119  _M_write(_Tp&& __x) const
4120  { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4121 
4122  public:
4123  _GLIBCXX_SIMD_INTRINSIC constexpr
4124  _SmartReference(_Up& __o, int __i) noexcept
4125  : _M_index(__i), _M_obj(__o) {}
4126 
4127  using value_type = _ValueType;
4128 
4129  _GLIBCXX_SIMD_INTRINSIC
4130  _SmartReference(const _SmartReference&) = delete;
4131 
4132  _GLIBCXX_SIMD_INTRINSIC constexpr
4133  operator value_type() const noexcept
4134  { return _M_read(); }
4135 
4136  template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4137  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4138  operator=(_Tp&& __x) &&
4139  {
4140  _M_write(static_cast<_Tp&&>(__x));
4141  return {_M_obj, _M_index};
4142  }
4143 
4144 #define _GLIBCXX_SIMD_OP_(__op) \
4145  template <typename _Tp, \
4146  typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4147  typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4148  typename = _ValuePreservingOrInt<_TT, value_type>> \
4149  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4150  operator __op##=(_Tp&& __x) && \
4151  { \
4152  const value_type& __lhs = _M_read(); \
4153  _M_write(__lhs __op __x); \
4154  return {_M_obj, _M_index}; \
4155  }
4156  _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4157  _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4158  _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4159 #undef _GLIBCXX_SIMD_OP_
4160 
4161  template <typename _Tp = void,
4162  typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4163  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4164  operator++() &&
4165  {
4166  value_type __x = _M_read();
4167  _M_write(++__x);
4168  return {_M_obj, _M_index};
4169  }
4170 
4171  template <typename _Tp = void,
4172  typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4173  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4174  operator++(int) &&
4175  {
4176  const value_type __r = _M_read();
4177  value_type __x = __r;
4178  _M_write(++__x);
4179  return __r;
4180  }
4181 
4182  template <typename _Tp = void,
4183  typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4184  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4185  operator--() &&
4186  {
4187  value_type __x = _M_read();
4188  _M_write(--__x);
4189  return {_M_obj, _M_index};
4190  }
4191 
4192  template <typename _Tp = void,
4193  typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4194  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4195  operator--(int) &&
4196  {
4197  const value_type __r = _M_read();
4198  value_type __x = __r;
4199  _M_write(--__x);
4200  return __r;
4201  }
4202 
4203  _GLIBCXX_SIMD_INTRINSIC friend void
4204  swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4205  conjunction<
4206  is_nothrow_constructible<value_type, _SmartReference&&>,
4207  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4208  {
4209  value_type __tmp = static_cast<_SmartReference&&>(__a);
4210  static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4211  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4212  }
4213 
4214  _GLIBCXX_SIMD_INTRINSIC friend void
4215  swap(value_type& __a, _SmartReference&& __b) noexcept(
4216  conjunction<
4217  is_nothrow_constructible<value_type, value_type&&>,
4218  is_nothrow_assignable<value_type&, value_type&&>,
4219  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4220  {
4221  value_type __tmp(std::move(__a));
4222  __a = static_cast<value_type>(__b);
4223  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4224  }
4225 
4226  _GLIBCXX_SIMD_INTRINSIC friend void
4227  swap(_SmartReference&& __a, value_type& __b) noexcept(
4228  conjunction<
4229  is_nothrow_constructible<value_type, _SmartReference&&>,
4230  is_nothrow_assignable<value_type&, value_type&&>,
4231  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4232  {
4233  value_type __tmp(__a);
4234  static_cast<_SmartReference&&>(__a) = std::move(__b);
4235  __b = std::move(__tmp);
4236  }
4237  };
4238 
4239 // }}}
4240 // __scalar_abi_wrapper {{{
4241 template <int _Bytes>
4242  struct __scalar_abi_wrapper
4243  {
4244  template <typename _Tp> static constexpr size_t _S_full_size = 1;
4245  template <typename _Tp> static constexpr size_t _S_size = 1;
4246  template <typename _Tp> static constexpr size_t _S_is_partial = false;
4247 
4248  template <typename _Tp, typename _Abi = simd_abi::scalar>
4249  static constexpr bool _S_is_valid_v
4250  = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4251  };
4252 
4253 // }}}
4254 // __decay_abi metafunction {{{
4255 template <typename _Tp>
4256  struct __decay_abi { using type = _Tp; };
4257 
4258 template <int _Bytes>
4259  struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4260  { using type = simd_abi::scalar; };
4261 
4262 // }}}
4263 // __find_next_valid_abi metafunction {{{1
4264 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4265 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4266 // recursion at 2 elements in the resulting ABI tag. In this case
4267 // type::_S_is_valid_v<_Tp> may be false.
4268 template <template <int> class _Abi, int _Bytes, typename _Tp>
4269  struct __find_next_valid_abi
4270  {
4271  static constexpr auto
4272  _S_choose()
4273  {
4274  constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4275  using _NextAbi = _Abi<_NextBytes>;
4276  if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4277  return _Abi<_Bytes>();
4278  else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4279  && _NextAbi::template _S_is_valid_v<_Tp>)
4280  return _NextAbi();
4281  else
4282  return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4283  }
4284 
4285  using type = decltype(_S_choose());
4286  };
4287 
4288 template <int _Bytes, typename _Tp>
4289  struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4290  { using type = simd_abi::scalar; };
4291 
4292 // _AbiList {{{1
4293 template <template <int> class...>
4294  struct _AbiList
4295  {
4296  template <typename, int> static constexpr bool _S_has_valid_abi = false;
4297  template <typename, int> using _FirstValidAbi = void;
4298  template <typename, int> using _BestAbi = void;
4299  };
4300 
4301 template <template <int> class _A0, template <int> class... _Rest>
4302  struct _AbiList<_A0, _Rest...>
4303  {
4304  template <typename _Tp, int _Np>
4305  static constexpr bool _S_has_valid_abi
4306  = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4307  _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4308 
4309  template <typename _Tp, int _Np>
4310  using _FirstValidAbi = conditional_t<
4311  _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4312  typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4313  typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4314 
4315  template <typename _Tp, int _Np>
4316  static constexpr auto
4317  _S_determine_best_abi()
4318  {
4319  static_assert(_Np >= 1);
4320  constexpr int _Bytes = sizeof(_Tp) * _Np;
4321  if constexpr (_Np == 1)
4322  return __make_dependent_t<_Tp, simd_abi::scalar>{};
4323  else
4324  {
4325  constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4326  // _A0<_Bytes> is good if:
4327  // 1. The ABI tag is valid for _Tp
4328  // 2. The storage overhead is no more than padding to fill the next
4329  // power-of-2 number of bytes
4330  if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4331  _Tp> && __fullsize / 2 < _Np)
4332  return typename __decay_abi<_A0<_Bytes>>::type{};
4333  else
4334  {
4335  using _Bp =
4336  typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4337  if constexpr (_Bp::template _S_is_valid_v<
4338  _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4339  return _Bp{};
4340  else
4341  return
4342  typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4343  }
4344  }
4345  }
4346 
4347  template <typename _Tp, int _Np>
4348  using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4349  };
4350 
4351 // }}}1
4352 
4353 // the following lists all native ABIs, which makes them accessible to
4354 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4355 // matters: Whatever comes first has higher priority.
4356 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4357  __scalar_abi_wrapper>;
4358 
4359 // valid _SimdTraits specialization {{{1
4360 template <typename _Tp, typename _Abi>
4361  struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4362  : _Abi::template __traits<_Tp> {};
4363 
4364 // __deduce_impl specializations {{{1
4365 // try all native ABIs (including scalar) first
4366 template <typename _Tp, size_t _Np>
4367  struct __deduce_impl<
4368  _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4369  { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4370 
4371 // fall back to fixed_size only if scalar and native ABIs don't match
4372 template <typename _Tp, size_t _Np, typename = void>
4373  struct __deduce_fixed_size_fallback {};
4374 
4375 template <typename _Tp, size_t _Np>
4376  struct __deduce_fixed_size_fallback<_Tp, _Np,
4377  enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4378  { using type = simd_abi::fixed_size<_Np>; };
4379 
4380 template <typename _Tp, size_t _Np, typename>
4381  struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4382 
4383 //}}}1
4384 /// @endcond
4385 
4386 // simd_mask {{{
4387 template <typename _Tp, typename _Abi>
4388  class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4389  {
4390  // types, tags, and friends {{{
4391  using _Traits = _SimdTraits<_Tp, _Abi>;
4392  using _MemberType = typename _Traits::_MaskMember;
4393 
4394  // We map all masks with equal element sizeof to a single integer type, the
4395  // one given by __int_for_sizeof_t<_Tp>. This is the approach
4396  // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4397  // template specializations in the implementation classes.
4398  using _Ip = __int_for_sizeof_t<_Tp>;
4399  static constexpr _Ip* _S_type_tag = nullptr;
4400 
4401  friend typename _Traits::_MaskBase;
4402  friend class simd<_Tp, _Abi>; // to construct masks on return
4403  friend typename _Traits::_SimdImpl; // to construct masks on return and
4404  // inspect data on masked operations
4405  public:
4406  using _Impl = typename _Traits::_MaskImpl;
4407  friend _Impl;
4408 
4409  // }}}
4410  // member types {{{
4411  using value_type = bool;
4412  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4413  using simd_type = simd<_Tp, _Abi>;
4414  using abi_type = _Abi;
4415 
4416  // }}}
4417  static constexpr size_t size() // {{{
4418  { return __size_or_zero_v<_Tp, _Abi>; }
4419 
4420  // }}}
4421  // constructors & assignment {{{
4422  simd_mask() = default;
4423  simd_mask(const simd_mask&) = default;
4424  simd_mask(simd_mask&&) = default;
4425  simd_mask& operator=(const simd_mask&) = default;
4426  simd_mask& operator=(simd_mask&&) = default;
4427 
4428  // }}}
4429  // access to internal representation (optional feature) {{{
4430  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit
4431  simd_mask(typename _Traits::_MaskCastType __init)
4432  : _M_data{__init} {}
4433  // conversions to internal type is done in _MaskBase
4434 
4435  // }}}
4436  // bitset interface (extension to be proposed) {{{
4437  // TS_FEEDBACK:
4438  // Conversion of simd_mask to and from bitset makes it much easier to
4439  // interface with other facilities. I suggest adding `static
4440  // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4441  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask
4442  __from_bitset(bitset<size()> bs)
4443  { return {__bitset_init, bs}; }
4444 
4445  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4446  __to_bitset() const
4447  { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4448 
4449  // }}}
4450  // explicit broadcast constructor {{{
4451  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4452  simd_mask(value_type __x)
4453  : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4454 
4455  // }}}
4456  // implicit type conversion constructor {{{
4457  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4458  // proposed improvement
4459  template <typename _Up, typename _A2,
4460  typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4461  _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4462  != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4463  simd_mask(const simd_mask<_Up, _A2>& __x)
4464  : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4465  #else
4466  // conforming to ISO/IEC 19570:2018
4467  template <typename _Up, typename = enable_if_t<conjunction<
4468  is_same<abi_type, simd_abi::fixed_size<size()>>,
4469  is_same<_Up, _Up>>::value>>
4470  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4471  simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4472  : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4473  #endif
4474 
4475  // }}}
4476  // load constructor {{{
4477  template <typename _Flags>
4478  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4479  simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>)
4480  : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4481 
4482  template <typename _Flags>
4483  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4484  simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4485  : _M_data{}
4486  {
4487  _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4488  _Flags::template _S_apply<simd_mask>(__mem));
4489  }
4490 
4491  // }}}
4492  // loads [simd_mask.load] {{{
4493  template <typename _Flags>
4494  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4495  copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>)
4496  { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4497 
4498  // }}}
4499  // stores [simd_mask.store] {{{
4500  template <typename _Flags>
4501  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4502  copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const
4503  { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4504 
4505  // }}}
4506  // scalar access {{{
4507  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4508  operator[](size_t __i)
4509  {
4510  if (__i >= size())
4511  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4512  return {_M_data, int(__i)};
4513  }
4514 
4515  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4516  operator[](size_t __i) const
4517  {
4518  if (__i >= size())
4519  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4520  if constexpr (__is_scalar_abi<_Abi>())
4521  return _M_data;
4522  else
4523  return static_cast<bool>(_M_data[__i]);
4524  }
4525 
4526  // }}}
4527  // negation {{{
4528  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4529  operator!() const
4530  { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4531 
4532  // }}}
4533  // simd_mask binary operators [simd_mask.binary] {{{
4534  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4535  // simd_mask<int> && simd_mask<uint> needs disambiguation
4536  template <typename _Up, typename _A2,
4537  typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4538  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4539  operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4540  {
4541  return {__private_init,
4542  _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4543  }
4544 
4545  template <typename _Up, typename _A2,
4546  typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4547  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4548  operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4549  {
4550  return {__private_init,
4551  _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4552  }
4553  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4554 
4555  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4556  operator&&(const simd_mask& __x, const simd_mask& __y)
4557  { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4558 
4559  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4560  operator||(const simd_mask& __x, const simd_mask& __y)
4561  { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4562 
4563  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4564  operator&(const simd_mask& __x, const simd_mask& __y)
4565  { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4566 
4567  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4568  operator|(const simd_mask& __x, const simd_mask& __y)
4569  { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4570 
4571  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4572  operator^(const simd_mask& __x, const simd_mask& __y)
4573  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4574 
4575  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4576  operator&=(simd_mask& __x, const simd_mask& __y)
4577  {
4578  __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4579  return __x;
4580  }
4581 
4582  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4583  operator|=(simd_mask& __x, const simd_mask& __y)
4584  {
4585  __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4586  return __x;
4587  }
4588 
4589  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4590  operator^=(simd_mask& __x, const simd_mask& __y)
4591  {
4592  __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4593  return __x;
4594  }
4595 
4596  // }}}
4597  // simd_mask compares [simd_mask.comparison] {{{
4598  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4599  operator==(const simd_mask& __x, const simd_mask& __y)
4600  { return !operator!=(__x, __y); }
4601 
4602  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4603  operator!=(const simd_mask& __x, const simd_mask& __y)
4604  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4605 
4606  // }}}
4607  // private_init ctor {{{
4608  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4609  simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4610  : _M_data(__init) {}
4611 
4612  // }}}
4613  // private_init generator ctor {{{
4614  template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4615  _GLIBCXX_SIMD_INTRINSIC constexpr
4616  simd_mask(_PrivateInit, _Fp&& __gen)
4617  : _M_data()
4618  {
4619  __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4620  _Impl::_S_set(_M_data, __i, __gen(__i));
4621  });
4622  }
4623 
4624  // }}}
4625  // bitset_init ctor {{{
4626  _GLIBCXX_SIMD_INTRINSIC constexpr
4627  simd_mask(_BitsetInit, bitset<size()> __init)
4628  : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4629  {}
4630 
4631  // }}}
4632  // __cvt {{{
4633  // TS_FEEDBACK:
4634  // The conversion operator this implements should be a ctor on simd_mask.
4635  // Once you call .__cvt() on a simd_mask it converts conveniently.
4636  // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4637  struct _CvtProxy
4638  {
4639  template <typename _Up, typename _A2,
4640  typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4641  operator simd_mask<_Up, _A2>() &&
4642  {
4643  using namespace std::experimental::__proposed;
4644  return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4645  }
4646 
4647  const simd_mask<_Tp, _Abi>& _M_data;
4648  };
4649 
4650  _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4651  __cvt() const
4652  { return {*this}; }
4653 
4654  // }}}
4655  // operator?: overloads (suggested extension) {{{
4656  #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4657  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4658  operator?:(const simd_mask& __k, const simd_mask& __where_true,
4659  const simd_mask& __where_false)
4660  {
4661  auto __ret = __where_false;
4662  _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4663  return __ret;
4664  }
4665 
4666  template <typename _U1, typename _U2,
4667  typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4668  typename = enable_if_t<conjunction_v<
4669  is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4670  is_convertible<simd_mask, typename _Rp::mask_type>>>>
4671  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4672  operator?:(const simd_mask& __k, const _U1& __where_true,
4673  const _U2& __where_false)
4674  {
4675  _Rp __ret = __where_false;
4676  _Rp::_Impl::_S_masked_assign(
4677  __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4678  __data(static_cast<_Rp>(__where_true)));
4679  return __ret;
4680  }
4681 
4682  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4683  template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4684  typename = enable_if_t<
4685  conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4686  is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4687  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4688  operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4689  const simd_mask<_Up, _Au>& __where_false)
4690  {
4691  simd_mask __ret = __where_false;
4692  _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4693  __where_true._M_data);
4694  return __ret;
4695  }
4696  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4697  #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4698 
4699  // }}}
4700  // _M_is_constprop {{{
4701  _GLIBCXX_SIMD_INTRINSIC constexpr bool
4702  _M_is_constprop() const
4703  {
4704  if constexpr (__is_scalar_abi<_Abi>())
4705  return __builtin_constant_p(_M_data);
4706  else
4707  return _M_data._M_is_constprop();
4708  }
4709 
4710  // }}}
4711 
4712  private:
4713  friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4714  friend auto& __data<_Tp, abi_type>(simd_mask&);
4715  alignas(_Traits::_S_mask_align) _MemberType _M_data;
4716  };
4717 
4718 // }}}
4719 
4720 /// @cond undocumented
4721 // __data(simd_mask) {{{
4722 template <typename _Tp, typename _Ap>
4723  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4724  __data(const simd_mask<_Tp, _Ap>& __x)
4725  { return __x._M_data; }
4726 
4727 template <typename _Tp, typename _Ap>
4728  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4729  __data(simd_mask<_Tp, _Ap>& __x)
4730  { return __x._M_data; }
4731 
4732 // }}}
4733 /// @endcond
4734 
4735 // simd_mask reductions [simd_mask.reductions] {{{
4736 template <typename _Tp, typename _Abi>
4737  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4738  all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4739  {
4740  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4741  {
4742  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4743  if (!__k[__i])
4744  return false;
4745  return true;
4746  }
4747  else
4748  return _Abi::_MaskImpl::_S_all_of(__k);
4749  }
4750 
4751 template <typename _Tp, typename _Abi>
4752  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4753  any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4754  {
4755  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4756  {
4757  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4758  if (__k[__i])
4759  return true;
4760  return false;
4761  }
4762  else
4763  return _Abi::_MaskImpl::_S_any_of(__k);
4764  }
4765 
4766 template <typename _Tp, typename _Abi>
4767  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4768  none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4769  {
4770  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4771  {
4772  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4773  if (__k[__i])
4774  return false;
4775  return true;
4776  }
4777  else
4778  return _Abi::_MaskImpl::_S_none_of(__k);
4779  }
4780 
4781 template <typename _Tp, typename _Abi>
4782  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4783  some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4784  {
4785  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4786  {
4787  for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4788  if (__k[__i] != __k[__i - 1])
4789  return true;
4790  return false;
4791  }
4792  else
4793  return _Abi::_MaskImpl::_S_some_of(__k);
4794  }
4795 
4796 template <typename _Tp, typename _Abi>
4797  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4798  popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4799  {
4800  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4801  {
4802  const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4803  __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4804  return ((__elements != 0) + ...);
4805  });
4806  if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4807  return __r;
4808  }
4809  return _Abi::_MaskImpl::_S_popcount(__k);
4810  }
4811 
4812 template <typename _Tp, typename _Abi>
4813  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4814  find_first_set(const simd_mask<_Tp, _Abi>& __k)
4815  {
4816  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4817  {
4818  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4819  const size_t _Idx = __call_with_n_evaluations<_Np>(
4820  [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4821  return std::min({__indexes...});
4822  }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4823  return __k[__i] ? +__i : _Np;
4824  });
4825  if (_Idx >= _Np)
4826  __invoke_ub("find_first_set(empty mask) is UB");
4827  if (__builtin_constant_p(_Idx))
4828  return _Idx;
4829  }
4830  return _Abi::_MaskImpl::_S_find_first_set(__k);
4831  }
4832 
4833 template <typename _Tp, typename _Abi>
4834  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4835  find_last_set(const simd_mask<_Tp, _Abi>& __k)
4836  {
4837  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4838  {
4839  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4840  const int _Idx = __call_with_n_evaluations<_Np>(
4841  [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4842  return std::max({__indexes...});
4843  }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4844  return __k[__i] ? int(__i) : -1;
4845  });
4846  if (_Idx < 0)
4847  __invoke_ub("find_first_set(empty mask) is UB");
4848  if (__builtin_constant_p(_Idx))
4849  return _Idx;
4850  }
4851  return _Abi::_MaskImpl::_S_find_last_set(__k);
4852  }
4853 
4854 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4855 all_of(_ExactBool __x) noexcept
4856 { return __x; }
4857 
4858 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4859 any_of(_ExactBool __x) noexcept
4860 { return __x; }
4861 
4862 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4863 none_of(_ExactBool __x) noexcept
4864 { return !__x; }
4865 
4866 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4867 some_of(_ExactBool) noexcept
4868 { return false; }
4869 
4870 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4871 popcount(_ExactBool __x) noexcept
4872 { return __x; }
4873 
4874 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4875 find_first_set(_ExactBool)
4876 { return 0; }
4877 
4878 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4879 find_last_set(_ExactBool)
4880 { return 0; }
4881 
4882 // }}}
4883 
4884 /// @cond undocumented
4885 // _SimdIntOperators{{{1
4886 template <typename _V, typename _Impl, bool>
4887  class _SimdIntOperators {};
4888 
4889 template <typename _V, typename _Impl>
4890  class _SimdIntOperators<_V, _Impl, true>
4891  {
4892  _GLIBCXX_SIMD_INTRINSIC constexpr const _V&
4893  __derived() const
4894  { return *static_cast<const _V*>(this); }
4895 
4896  template <typename _Tp>
4897  _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4898  _S_make_derived(_Tp&& __d)
4899  { return {__private_init, static_cast<_Tp&&>(__d)}; }
4900 
4901  public:
4902  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4903  { return __lhs = __lhs % __x; }
4904 
4905  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4906  { return __lhs = __lhs & __x; }
4907 
4908  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4909  { return __lhs = __lhs | __x; }
4910 
4911  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4912  { return __lhs = __lhs ^ __x; }
4913 
4914  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4915  { return __lhs = __lhs << __x; }
4916 
4917  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4918  { return __lhs = __lhs >> __x; }
4919 
4920  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4921  { return __lhs = __lhs << __x; }
4922 
4923  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4924  { return __lhs = __lhs >> __x; }
4925 
4926  _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4927  {
4928  return _SimdIntOperators::_S_make_derived(
4929  _Impl::_S_modulus(__data(__x), __data(__y)));
4930  }
4931 
4932  _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4933  {
4934  return _SimdIntOperators::_S_make_derived(
4935  _Impl::_S_bit_and(__data(__x), __data(__y)));
4936  }
4937 
4938  _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4939  {
4940  return _SimdIntOperators::_S_make_derived(
4941  _Impl::_S_bit_or(__data(__x), __data(__y)));
4942  }
4943 
4944  _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4945  {
4946  return _SimdIntOperators::_S_make_derived(
4947  _Impl::_S_bit_xor(__data(__x), __data(__y)));
4948  }
4949 
4950  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4951  {
4952  return _SimdIntOperators::_S_make_derived(
4953  _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4954  }
4955 
4956  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4957  {
4958  return _SimdIntOperators::_S_make_derived(
4959  _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4960  }
4961 
4962  template <typename _VV = _V>
4963  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4964  {
4965  using _Tp = typename _VV::value_type;
4966  if (__y < 0)
4967  __invoke_ub("The behavior is undefined if the right operand of a "
4968  "shift operation is negative. [expr.shift]\nA shift by "
4969  "%d was requested",
4970  __y);
4971  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4972  __invoke_ub(
4973  "The behavior is undefined if the right operand of a "
4974  "shift operation is greater than or equal to the width of the "
4975  "promoted left operand. [expr.shift]\nA shift by %d was requested",
4976  __y);
4977  return _SimdIntOperators::_S_make_derived(
4978  _Impl::_S_bit_shift_left(__data(__x), __y));
4979  }
4980 
4981  template <typename _VV = _V>
4982  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4983  {
4984  using _Tp = typename _VV::value_type;
4985  if (__y < 0)
4986  __invoke_ub(
4987  "The behavior is undefined if the right operand of a shift "
4988  "operation is negative. [expr.shift]\nA shift by %d was requested",
4989  __y);
4990  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4991  __invoke_ub(
4992  "The behavior is undefined if the right operand of a shift "
4993  "operation is greater than or equal to the width of the promoted "
4994  "left operand. [expr.shift]\nA shift by %d was requested",
4995  __y);
4996  return _SimdIntOperators::_S_make_derived(
4997  _Impl::_S_bit_shift_right(__data(__x), __y));
4998  }
4999 
5000  // unary operators (for integral _Tp)
5001  _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
5002  { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5003  };
5004 
5005 //}}}1
5006 /// @endcond
5007 
5008 // simd {{{
5009 template <typename _Tp, typename _Abi>
5010  class simd : public _SimdIntOperators<
5011  simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
5012  conjunction<is_integral<_Tp>,
5013  typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5014  public _SimdTraits<_Tp, _Abi>::_SimdBase
5015  {
5016  using _Traits = _SimdTraits<_Tp, _Abi>;
5017  using _MemberType = typename _Traits::_SimdMember;
5018  using _CastType = typename _Traits::_SimdCastType;
5019  static constexpr _Tp* _S_type_tag = nullptr;
5020  friend typename _Traits::_SimdBase;
5021 
5022  public:
5023  using _Impl = typename _Traits::_SimdImpl;
5024  friend _Impl;
5025  friend _SimdIntOperators<simd, _Impl, true>;
5026 
5027  using value_type = _Tp;
5028  using reference = _SmartReference<_MemberType, _Impl, value_type>;
5029  using mask_type = simd_mask<_Tp, _Abi>;
5030  using abi_type = _Abi;
5031 
5032  static constexpr size_t size()
5033  { return __size_or_zero_v<_Tp, _Abi>; }
5034 
5035  _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5036  _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5037  _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5038  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5039  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5040 
5041  // implicit broadcast constructor
5042  template <typename _Up,
5043  typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5044  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5045  simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5046  : _M_data(
5047  _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5048  {}
5049 
5050  // implicit type conversion constructor (convert from fixed_size to
5051  // fixed_size)
5052  template <typename _Up>
5053  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5054  simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5055  enable_if_t<
5056  conjunction<
5057  is_same<simd_abi::fixed_size<size()>, abi_type>,
5058  negation<__is_narrowing_conversion<_Up, value_type>>,
5059  __converts_to_higher_integer_rank<_Up, value_type>>::value,
5060  void*> = nullptr)
5061  : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5062 
5063  // explicit type conversion constructor
5064 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5065  template <typename _Up, typename _A2,
5066  typename = decltype(static_simd_cast<simd>(
5067  declval<const simd<_Up, _A2>&>()))>
5068  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5069  simd(const simd<_Up, _A2>& __x)
5070  : simd(static_simd_cast<simd>(__x)) {}
5071 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5072 
5073  // generator constructor
5074  template <typename _Fp>
5075  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5076  simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5077  declval<_SizeConstant<0>&>())),
5078  value_type>* = nullptr)
5079  : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5080 
5081  // load constructor
5082  template <typename _Up, typename _Flags>
5083  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5084  simd(const _Up* __mem, _IsSimdFlagType<_Flags>)
5085  : _M_data(
5086  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5087  {}
5088 
5089  // loads [simd.load]
5090  template <typename _Up, typename _Flags>
5091  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5092  copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5093  {
5094  _M_data = static_cast<decltype(_M_data)>(
5095  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5096  }
5097 
5098  // stores [simd.store]
5099  template <typename _Up, typename _Flags>
5100  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5101  copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const
5102  {
5103  _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5104  _S_type_tag);
5105  }
5106 
5107  // scalar access
5108  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5109  operator[](size_t __i)
5110  { return {_M_data, int(__i)}; }
5111 
5112  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5113  operator[]([[maybe_unused]] size_t __i) const
5114  {
5115  if constexpr (__is_scalar_abi<_Abi>())
5116  {
5117  _GLIBCXX_DEBUG_ASSERT(__i == 0);
5118  return _M_data;
5119  }
5120  else
5121  return _M_data[__i];
5122  }
5123 
5124  // increment and decrement:
5125  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5126  operator++()
5127  {
5128  _Impl::_S_increment(_M_data);
5129  return *this;
5130  }
5131 
5132  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5133  operator++(int)
5134  {
5135  simd __r = *this;
5136  _Impl::_S_increment(_M_data);
5137  return __r;
5138  }
5139 
5140  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5141  operator--()
5142  {
5143  _Impl::_S_decrement(_M_data);
5144  return *this;
5145  }
5146 
5147  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5148  operator--(int)
5149  {
5150  simd __r = *this;
5151  _Impl::_S_decrement(_M_data);
5152  return __r;
5153  }
5154 
5155  // unary operators (for any _Tp)
5156  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5157  operator!() const
5158  { return {__private_init, _Impl::_S_negate(_M_data)}; }
5159 
5160  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5161  operator+() const
5162  { return *this; }
5163 
5164  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5165  operator-() const
5166  { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5167 
5168  // access to internal representation (suggested extension)
5169  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5170  simd(_CastType __init) : _M_data(__init) {}
5171 
5172  // compound assignment [simd.cassign]
5173  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5174  operator+=(simd& __lhs, const simd& __x)
5175  { return __lhs = __lhs + __x; }
5176 
5177  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5178  operator-=(simd& __lhs, const simd& __x)
5179  { return __lhs = __lhs - __x; }
5180 
5181  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5182  operator*=(simd& __lhs, const simd& __x)
5183  { return __lhs = __lhs * __x; }
5184 
5185  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5186  operator/=(simd& __lhs, const simd& __x)
5187  { return __lhs = __lhs / __x; }
5188 
5189  // binary operators [simd.binary]
5190  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5191  operator+(const simd& __x, const simd& __y)
5192  { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5193 
5194  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5195  operator-(const simd& __x, const simd& __y)
5196  { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5197 
5198  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5199  operator*(const simd& __x, const simd& __y)
5200  { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5201 
5202  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5203  operator/(const simd& __x, const simd& __y)
5204  { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5205 
5206  // compares [simd.comparison]
5207  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5208  operator==(const simd& __x, const simd& __y)
5209  { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5210 
5211  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5212  operator!=(const simd& __x, const simd& __y)
5213  {
5214  return simd::_S_make_mask(
5215  _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5216  }
5217 
5218  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5219  operator<(const simd& __x, const simd& __y)
5220  { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5221 
5222  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5223  operator<=(const simd& __x, const simd& __y)
5224  {
5225  return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5226  }
5227 
5228  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5229  operator>(const simd& __x, const simd& __y)
5230  { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5231 
5232  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5233  operator>=(const simd& __x, const simd& __y)
5234  {
5235  return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5236  }
5237 
5238  // operator?: overloads (suggested extension) {{{
5239 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5240  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5241  operator?:(const mask_type& __k, const simd& __where_true,
5242  const simd& __where_false)
5243  {
5244  auto __ret = __where_false;
5245  _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5246  return __ret;
5247  }
5248 
5249 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5250  // }}}
5251 
5252  // "private" because of the first arguments's namespace
5253  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5254  simd(_PrivateInit, const _MemberType& __init)
5255  : _M_data(__init) {}
5256 
5257  // "private" because of the first arguments's namespace
5258  _GLIBCXX_SIMD_INTRINSIC
5259  simd(_BitsetInit, bitset<size()> __init) : _M_data()
5260  { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5261 
5262  _GLIBCXX_SIMD_INTRINSIC constexpr bool
5263  _M_is_constprop() const
5264  {
5265  if constexpr (__is_scalar_abi<_Abi>())
5266  return __builtin_constant_p(_M_data);
5267  else
5268  return _M_data._M_is_constprop();
5269  }
5270 
5271  private:
5272  _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type
5273  _S_make_mask(typename mask_type::_MemberType __k)
5274  { return {__private_init, __k}; }
5275 
5276  friend const auto& __data<value_type, abi_type>(const simd&);
5277  friend auto& __data<value_type, abi_type>(simd&);
5278  alignas(_Traits::_S_simd_align) _MemberType _M_data;
5279  };
5280 
5281 // }}}
5282 /// @cond undocumented
5283 // __data {{{
5284 template <typename _Tp, typename _Ap>
5285  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5286  __data(const simd<_Tp, _Ap>& __x)
5287  { return __x._M_data; }
5288 
5289 template <typename _Tp, typename _Ap>
5290  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5291  __data(simd<_Tp, _Ap>& __x)
5292  { return __x._M_data; }
5293 
5294 // }}}
5295 namespace __float_bitwise_operators { //{{{
5296 template <typename _Tp, typename _Ap>
5297  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5298  operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5299  { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5300 
5301 template <typename _Tp, typename _Ap>
5302  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5303  operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5304  { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5305 
5306 template <typename _Tp, typename _Ap>
5307  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5308  operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5309  { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5310 } // namespace __float_bitwise_operators }}}
5311 /// @endcond
5312 
5313 /// @}
5314 _GLIBCXX_SIMD_END_NAMESPACE
5315 
5316 #endif // __cplusplus >= 201703L
5317 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5318 
5319 // vim: foldmethod=marker foldmarker={{{,}}}
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:706
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1540
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1453
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:86
Definition: simd.h:207
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition: range_access.h:290
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:729
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2579
Definition: simd.h:85
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition: valarray:1239
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:289
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
Definition: stl_algo.h:3301
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: tuple:1732
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
Definition: type_traits:1570
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:1974
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:245
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition: valarray:1217
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:83
bitset< _Nb > operator &(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1435
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2583
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
Definition: stl_algo.h:3656
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
Definition: chrono:1016
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1444
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
Definition: chrono:660
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
Definition: chrono:700
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1472