#ifndef VC_COMMON_ALGORITHMS_H_
#define VC_COMMON_ALGORITHMS_H_ 
#ifndef VC_COMMON_SIMDIZE_H_
#define VC_COMMON_SIMDIZE_H_ 
#include <tuple>
#include <array>
#ifndef VC_ALLOCATOR_H_
#define VC_ALLOCATOR_H_ 
#include <new>
#include <cstddef>
#include <cstdlib>
#include <utility>
#ifdef Vc_MSVC
#define Vc_DECLARE_ALLOCATOR(Type) \
namespace std \
{ \
template <> class allocator<Type> : public ::Vc::Allocator<Type> \
{ \
public: \
template <typename U> struct rebind { \
typedef ::std::allocator<U> other; \
}; \
\
const allocator &select_on_container_copy_construction() const { return *this; } \
}; \
}
#else
#define Vc_DECLARE_ALLOCATOR(Type) \
namespace std \
{ \
template <> class allocator<Type> : public ::Vc::Allocator<Type> \
{ \
public: \
template <typename U> struct rebind { \
typedef ::std::allocator<U> other; \
}; \
}; \
}
#endif
namespace Vc_VERSIONED_NAMESPACE
{
using std::size_t;
using std::ptrdiff_t;
template<typename T> class Allocator
{
private:
enum Constants {
#ifdef Vc_HAVE_STD_MAX_ALIGN_T
NaturalAlignment = alignof(std::max_align_t),
#elif defined(Vc_HAVE_MAX_ALIGN_T)
NaturalAlignment = alignof(::max_align_t),
#else
NaturalAlignment = sizeof(void *) > alignof(long double) ? sizeof(void *) :
(alignof(long double) > alignof(long long) ? alignof(long double) : alignof(long long)),
#endif
#if defined Vc_IMPL_AVX
SimdAlignment = 32,
#elif defined Vc_IMPL_SSE
SimdAlignment = 16,
#else
SimdAlignment = 1,
#endif
Alignment = alignof(T) > SimdAlignment ? alignof(T) : SimdAlignment,
ExtraBytes = Alignment > NaturalAlignment ? Alignment : 0,
AlignmentMask = Alignment - 1
};
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<typename U> struct rebind { typedef Allocator<U> other; };
Allocator() throw() { }
Allocator(const Allocator&) throw() { }
template<typename U> Allocator(const Allocator<U>&) throw() { }
pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const { return &x; }
pointer allocate(size_type n, const void* = 0)
{
if (n > this->max_size()) {
throw std::bad_alloc();
}
char *p = static_cast<char *>(::operator new(n * sizeof(T) + ExtraBytes));
if (ExtraBytes > 0) {
char *const pp = p;
p += ExtraBytes;
const char *null = 0;
p -= ((p - null) & AlignmentMask);
reinterpret_cast<char **>(p)[-1] = pp;
}
return reinterpret_cast<pointer>(p);
}
void deallocate(pointer p, size_type)
{
if (ExtraBytes > 0) {
p = reinterpret_cast<pointer *>(p)[-1];
}
::operator delete(p);
}
size_type max_size() const throw() { return size_t(-1) / sizeof(T); }
#ifdef Vc_MSVC
const Allocator &select_on_container_copy_construction() const { return *this; }
void construct(pointer p) { ::new(p) T(); }
void construct(pointer p, const T& val) { ::new(p) T(val); }
void destroy(pointer p) { p->~T(); }
#else
template<typename U, typename... Args> void construct(U* p, Args&&... args)
{
::new(p) U(std::forward<Args>(args)...);
}
template<typename U> void destroy(U* p) { p->~U(); }
#endif
};
template<typename T> inline bool operator==(const Allocator<T>&, const Allocator<T>&) { return true; }
template<typename T> inline bool operator!=(const Allocator<T>&, const Allocator<T>&) { return false; }
}
namespace std
{
template<typename T, typename Abi>
class allocator<Vc::Vector<T, Abi> > : public ::Vc::Allocator<Vc::Vector<T, Abi> >
{
public:
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
#ifdef Vc_MSVC
const allocator &select_on_container_copy_construction() const { return *this; }
#endif
};
template <typename T, typename Abi>
class allocator<Vc::Mask<T, Abi>> : public ::Vc::Allocator<Vc::Mask<T, Abi>>
{
public:
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
#ifdef Vc_MSVC
const allocator &select_on_container_copy_construction() const { return *this; }
#endif
};
template <typename T, std::size_t N, typename V, std::size_t M>
class allocator<Vc::SimdArray<T, N, V, M>> : public ::Vc::Allocator<Vc::SimdArray<T, N, V, M>>
{
public:
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
#ifdef Vc_MSVC
const allocator &select_on_container_copy_construction() const { return *this; }
#endif
};
template <typename T, std::size_t N, typename V, std::size_t M>
class allocator<Vc::SimdMaskArray<T, N, V, M>> : public ::Vc::Allocator<Vc::SimdMaskArray<T, N, V, M>>
{
public:
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
#ifdef Vc_MSVC
const allocator &select_on_container_copy_construction() const { return *this; }
#endif
};
}
#endif
#ifndef VC_COMMON_INTERLEAVEDMEMORY_H_
#define VC_COMMON_INTERLEAVEDMEMORY_H_ 
namespace Vc_VERSIONED_NAMESPACE
{
namespace Common
{
template<typename V, typename I, bool Readonly> struct InterleavedMemoryAccessBase
{
typedef typename std::conditional<
Readonly, typename std::add_const<typename V::EntryType>::type,
typename V::EntryType>::type T;
typedef typename V::AsArg VArg;
typedef T Ta Vc_MAY_ALIAS;
const I m_indexes;
Ta *const m_data;
Vc_ALWAYS_INLINE InterleavedMemoryAccessBase(typename I::AsArg indexes, Ta *data)
: m_indexes(indexes), m_data(data)
{
}
template <typename... Vs> Vc_INTRINSIC void deinterleave(Vs &&... vs) const
{
Impl::deinterleave(m_data, m_indexes, std::forward<Vs>(vs)...);
}
protected:
using Impl = Vc::Detail::InterleaveImpl<V, V::Size, sizeof(V)>;
template <typename T, std::size_t... Indexes>
Vc_INTRINSIC void callInterleave(T &&a, index_sequence<Indexes...>)
{
Impl::interleave(m_data, m_indexes, a[Indexes]...);
}
};
template <size_t StructSize, typename V, typename I = typename V::IndexType,
bool Readonly>
struct InterleavedMemoryReadAccess : public InterleavedMemoryAccessBase<V, I, Readonly>
{
typedef InterleavedMemoryAccessBase<V, I, Readonly> Base;
typedef typename Base::Ta Ta;
Vc_ALWAYS_INLINE InterleavedMemoryReadAccess(Ta *data, typename I::AsArg indexes)
: Base(StructSize == 1u
? indexes
: StructSize == 2u
? indexes << 1
: StructSize == 4u
? indexes << 2
: StructSize == 8u
? indexes << 3
: StructSize == 16u ? indexes << 4
: indexes * I(int(StructSize)),
data)
{
}
template <typename T, std::size_t... Indexes>
Vc_ALWAYS_INLINE T deinterleave_unpack(index_sequence<Indexes...>) const
{
T r;
Base::Impl::deinterleave(this->m_data, this->m_indexes, std::get<Indexes>(r)...);
return r;
}
template <typename T,
typename = enable_if<(std::is_default_constructible<T>::value &&
std::is_same<V, Traits::decay<decltype(std::get<0>(
std::declval<T &>()))>>::value)>>
Vc_ALWAYS_INLINE operator T() const
{
return deinterleave_unpack<T>(make_index_sequence<std::tuple_size<T>::value>());
}
};
template<typename I> struct CheckIndexesUnique
{
#ifdef NDEBUG
static Vc_INTRINSIC void test(const I &) {}
#else
static void test(const I &indexes)
{
const I test = indexes.sorted();
Vc_ASSERT(I::Size == 1 || (test == test.rotated(1)).isEmpty())
}
#endif
};
template<size_t S> struct CheckIndexesUnique<SuccessiveEntries<S> >
{
static Vc_INTRINSIC void test(const SuccessiveEntries<S> &) {}
};
template <size_t StructSize, typename V, typename I = typename V::IndexType>
struct InterleavedMemoryAccess : public InterleavedMemoryReadAccess<StructSize, V, I, false>
{
typedef InterleavedMemoryAccessBase<V, I, false> Base;
typedef typename Base::Ta Ta;
Vc_ALWAYS_INLINE InterleavedMemoryAccess(Ta *data, typename I::AsArg indexes)
: InterleavedMemoryReadAccess<StructSize, V, I, false>(data, indexes)
{
CheckIndexesUnique<I>::test(indexes);
}
template <int N> Vc_ALWAYS_INLINE void operator=(VectorReferenceArray<N, V> &&rhs)
{
static_assert(N <= StructSize,
"You_are_trying_to_scatter_more_data_into_the_struct_than_it_has");
this->callInterleave(std::move(rhs), make_index_sequence<N>());
}
template <int N> Vc_ALWAYS_INLINE void operator=(VectorReferenceArray<N, const V> &&rhs)
{
static_assert(N <= StructSize,
"You_are_trying_to_scatter_more_data_into_the_struct_than_it_has");
this->callInterleave(std::move(rhs), make_index_sequence<N>());
}
};
template<typename S, typename V> class InterleavedMemoryWrapper
{
typedef typename std::conditional<std::is_const<S>::value,
const typename V::EntryType,
typename V::EntryType>::type T;
typedef typename V::IndexType I;
typedef typename V::AsArg VArg;
typedef const I &IndexType;
static constexpr std::size_t StructSize = sizeof(S) / sizeof(T);
using ReadAccess = InterleavedMemoryReadAccess<StructSize, V>;
using Access =
typename std::conditional<std::is_const<T>::value, ReadAccess,
InterleavedMemoryAccess<StructSize, V>>::type;
using ReadSuccessiveEntries =
InterleavedMemoryReadAccess<StructSize, V, SuccessiveEntries<StructSize>>;
using AccessSuccessiveEntries = typename std::conditional<
std::is_const<T>::value, ReadSuccessiveEntries,
InterleavedMemoryAccess<StructSize, V, SuccessiveEntries<StructSize>>>::type;
typedef T Ta Vc_MAY_ALIAS;
Ta *const m_data;
static_assert(StructSize * sizeof(T) == sizeof(S),
"InterleavedMemoryAccess_does_not_support_packed_structs");
public:
Vc_ALWAYS_INLINE InterleavedMemoryWrapper(S *s)
: m_data(reinterpret_cast<Ta *>(s))
{
}
template <typename IT>
Vc_ALWAYS_INLINE enable_if<!std::is_convertible<IT, size_t>::value &&
std::is_convertible<IT, IndexType>::value &&
!std::is_const<S>::value,
Access>
operator[](IT indexes)
{
return Access(m_data, indexes);
}
Vc_ALWAYS_INLINE ReadAccess operator[](IndexType indexes) const
{
return ReadAccess(m_data, indexes);
}
Vc_ALWAYS_INLINE ReadAccess gather(IndexType indexes) const { return operator[](indexes); }
Vc_ALWAYS_INLINE ReadSuccessiveEntries operator[](size_t first) const
{
return ReadSuccessiveEntries(m_data, first);
}
Vc_ALWAYS_INLINE AccessSuccessiveEntries operator[](size_t first)
{
return AccessSuccessiveEntries(m_data, first);
}
};
}
using Common::InterleavedMemoryWrapper;
template <typename V, typename S>
inline Common::InterleavedMemoryWrapper<S, V> make_interleave_wrapper(S *s)
{
return Common::InterleavedMemoryWrapper<S, V>(s);
}
}
#endif
namespace Vc_VERSIONED_NAMESPACE
{
namespace SimdizeDetail
{
using std::is_same;
using std::is_base_of;
using std::false_type;
using std::true_type;
using std::iterator_traits;
using std::conditional;
using std::size_t;
template <typename... Ts> struct Typelist;
enum class Category {
None,
ArithmeticVectorizable,
InputIterator,
OutputIterator,
ForwardIterator,
BidirectionalIterator,
RandomAccessIterator,
ClassTemplate
};
template <typename T, typename ItCat = typename T::iterator_category>
constexpr Category iteratorCategories(int, ItCat * = nullptr)
{
return is_base_of<std::random_access_iterator_tag, ItCat>::value
? Category::RandomAccessIterator
: is_base_of<std::bidirectional_iterator_tag, ItCat>::value
? Category::BidirectionalIterator
: is_base_of<std::forward_iterator_tag, ItCat>::value
? Category::ForwardIterator
: is_base_of<std::output_iterator_tag, ItCat>::value
? Category::OutputIterator
: is_base_of<std::input_iterator_tag, ItCat>::value
? Category::InputIterator
: Category::None;
}
template <typename T>
constexpr enable_if<std::is_pointer<T>::value, Category> iteratorCategories(float)
{
return Category::RandomAccessIterator;
}
template <typename T> constexpr Category iteratorCategories(...)
{
return Category::None;
}
template <typename T> struct is_class_template : public false_type
{
};
template <template <typename...> class C, typename... Ts>
struct is_class_template<C<Ts...>> : public true_type
{
};
template <typename T> constexpr Category typeCategory()
{
return (is_same<T, bool>::value || is_same<T, short>::value ||
is_same<T, unsigned short>::value || is_same<T, int>::value ||
is_same<T, unsigned int>::value || is_same<T, float>::value ||
is_same<T, double>::value)
? Category::ArithmeticVectorizable
: iteratorCategories<T>(int()) != Category::None
? iteratorCategories<T>(int())
: is_class_template<T>::value ? Category::ClassTemplate
: Category::None;
}
template <typename T, size_t TupleSize = std::tuple_size<T>::value>
constexpr size_t determine_tuple_size()
{
return TupleSize;
}
template <typename T, size_t TupleSize = T::tuple_size>
constexpr size_t determine_tuple_size(size_t = T::tuple_size)
{
return TupleSize;
}
template <typename T> struct determine_tuple_size_
: public std::integral_constant<size_t, determine_tuple_size<T>()>
{};
namespace
{
template <typename T> struct The_simdization_for_the_requested_type_is_not_implemented;
}
template <typename T, size_t N, typename MT, Category = typeCategory<T>()>
struct ReplaceTypes : public The_simdization_for_the_requested_type_is_not_implemented<T>
{
};
template <typename T, size_t N, typename MT> struct ReplaceTypes<T, N, MT, Category::None>
{
typedef T type;
};
template <typename T, size_t N = 0, typename MT = void>
using simdize = typename SimdizeDetail::ReplaceTypes<T, N, MT>::type;
template <class T, size_t N,
class Best = typename Common::select_best_vector_type<T, N>::type>
using deduce_vector_t =
typename std::conditional<Best::size() == N, Best, SimdArray<T, N>>::type;
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::ArithmeticVectorizable>
: public conditional<N == 0, Vector<T>, deduce_vector_t<T, N>> {
};
template <size_t N, typename MT>
struct ReplaceTypes<bool, N, MT, Category::ArithmeticVectorizable>
: public std::enable_if<true, typename ReplaceTypes<MT, N, MT>::type::mask_type> {
};
template <size_t N>
struct ReplaceTypes<bool, N, void, Category::ArithmeticVectorizable>
: public ReplaceTypes<bool, N, float, Category::ArithmeticVectorizable>
{
};
template <size_t N, typename MT, typename Replaced, typename... Remaining>
struct SubstituteOneByOne;
template <size_t N, typename MT, typename... Replaced, typename T,
typename... Remaining>
struct SubstituteOneByOne<N, MT, Typelist<Replaced...>, T, Remaining...>
{
private:
template <typename U, size_t M = U::Size>
static std::integral_constant<size_t, M> size_or_0(int);
template <typename U> static std::integral_constant<size_t, 0> size_or_0(...);
using V = simdize<T, N, MT>;
static constexpr auto NewN = N != 0 ? N : decltype(size_or_0<V>(int()))::value;
typedef conditional_t<(N != NewN && is_same<MT, void>::value),
conditional_t<is_same<T, bool>::value, float, T>, MT> NewMT;
public:
using type = typename SubstituteOneByOne<NewN, NewMT, Typelist<Replaced..., V>,
Remaining...>::type;
};
template <size_t Size, typename... Replaced> struct SubstitutedBase;
template <typename Replaced> struct SubstitutedBase<1, Replaced> {
template <typename ValueT, template <typename, ValueT...> class C, ValueT... Values>
using SubstitutedWithValues = C<Replaced, Values...>;
};
template <typename R0, typename R1> struct SubstitutedBase<2, R0, R1>
{
template <typename ValueT, template <typename, typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<R0, R1, Values...>;
};
template <typename R0, typename R1, typename R2> struct SubstitutedBase<3, R0, R1, R2>
{
template <typename ValueT, template <typename, typename, typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<R0, R1, R2, Values...>;
};
#if defined Vc_ICC || defined Vc_MSVC
#define Vc_VALUE_PACK_EXPANSION_IS_BROKEN 1
#endif
template <typename... Replaced> struct SubstitutedBase<4, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
template <typename ValueT,
template <typename, typename, typename, typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<Replaced..., Values...>;
#endif
};
template <typename... Replaced> struct SubstitutedBase<5, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
template <typename ValueT, template <typename, typename, typename, typename, typename,
ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<Replaced..., Values...>;
#endif
};
template <typename... Replaced> struct SubstitutedBase<6, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
template <typename ValueT, template <typename, typename, typename, typename, typename,
typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<Replaced..., Values...>;
#endif
};
template <typename... Replaced> struct SubstitutedBase<7, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
template <typename ValueT, template <typename, typename, typename, typename, typename,
typename, typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<Replaced..., Values...>;
#endif
};
template <typename... Replaced> struct SubstitutedBase<8, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
template <typename ValueT, template <typename, typename, typename, typename, typename,
typename, typename, typename, ValueT...> class C,
ValueT... Values>
using SubstitutedWithValues = C<Replaced..., Values...>;
#endif
};
template <size_t N_, typename MT, typename Replaced0, typename... Replaced>
struct SubstituteOneByOne<N_, MT, Typelist<Replaced0, Replaced...>>
{
struct type
: public SubstitutedBase<sizeof...(Replaced) + 1, Replaced0, Replaced...> {
static constexpr auto N = N_;
template <template <typename...> class C>
using Substituted = C<Replaced0, Replaced...>;
};
};
template <typename Scalar, typename Base, size_t N> class Adapter;
template <template <typename...> class C, typename... Ts, size_t N, typename MT>
struct ReplaceTypes<C<Ts...>, N, MT, Category::ClassTemplate>
{
using SubstitutionResult =
typename SubstituteOneByOne<N, MT, Typelist<>, Ts...>::type;
using Vectorized = typename SubstitutionResult::template Substituted<C>;
using type = conditional_t<is_same<C<Ts...>, Vectorized>::value, C<Ts...>,
Adapter<C<Ts...>, Vectorized, SubstitutionResult::N>>;
};
#ifdef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_) \
template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, ValueType_...> class C, typename T0, \
typename T1, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, ValueType_...> \
class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, typename, \
ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
typename T5, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, typename, \
typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
typename T5, typename T6, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, ValueType_> class C, typename T0, ValueType_ Value0, \
size_t N, typename MT> \
struct ReplaceTypes<C<T0, Value0>, N, MT, Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t<is_same<C<T0, Value0>, Substituted>::value, C<T0, Value0>, \
Adapter<C<T0, Value0>, Substituted, NN>> type; \
}; \
template <template <typename, typename, ValueType_> class C, typename T0, \
typename T1, ValueType_ Value0, size_t N, typename MT> \
struct ReplaceTypes<C<T0, T1, Value0>, N, MT, Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t<is_same<C<T0, T1, Value0>, Substituted>::value, \
C<T0, T1, Value0>, \
Adapter<C<T0, T1, Value0>, Substituted, NN>> type; \
}; \
template <template <typename, typename, typename, ValueType_> class C, typename T0, \
typename T1, typename T2, ValueType_ Value0, size_t N, typename MT> \
struct ReplaceTypes<C<T0, T1, T2, Value0>, N, MT, Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t<is_same<C<T0, T1, T2, Value0>, Substituted>::value, \
C<T0, T1, T2, Value0>, \
Adapter<C<T0, T1, T2, Value0>, Substituted, NN>> type; \
}
#else
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_) \
template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, ValueType_...> class C, typename T0, \
typename T1, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, ValueType_ Value0, \
ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, ValueType_...> \
class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, typename, \
ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
typename T5, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, typename, typename, typename, typename, typename, \
typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, typename T3, typename T4, \
typename T5, typename T6, ValueType_ Value0, ValueType_... Values> \
struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>> \
: public true_type { \
}; \
template <template <typename, ValueType_...> class C, typename T0, \
ValueType_ Value0, ValueType_... Values, size_t N, typename MT> \
struct ReplaceTypes<C<T0, Value0, Values...>, N, MT, Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
Values...> Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t<is_same<C<T0, Value0, Values...>, Substituted>::value, \
C<T0, Value0, Values...>, \
Adapter<C<T0, Value0, Values...>, Substituted, NN>> type; \
}; \
template <template <typename, typename, ValueType_...> class C, typename T0, \
typename T1, ValueType_ Value0, ValueType_... Values, size_t N, \
typename MT> \
struct ReplaceTypes<C<T0, T1, Value0, Values...>, N, MT, Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
Values...> Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t<is_same<C<T0, T1, Value0, Values...>, Substituted>::value, \
C<T0, T1, Value0, Values...>, \
Adapter<C<T0, T1, Value0, Values...>, Substituted, NN>> \
type; \
}; \
template <template <typename, typename, typename, ValueType_...> class C, \
typename T0, typename T1, typename T2, ValueType_ Value0, \
ValueType_... Values, size_t N, typename MT> \
struct ReplaceTypes<C<T0, T1, T2, Value0, Values...>, N, MT, \
Category::ClassTemplate> { \
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp; \
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
Values...> Substituted; \
static constexpr auto NN = tmp::N; \
typedef conditional_t< \
is_same<C<T0, T1, T2, Value0, Values...>, Substituted>::value, \
C<T0, T1, T2, Value0, Values...>, \
Adapter<C<T0, T1, T2, Value0, Values...>, Substituted, NN>> type; \
}
#endif
Vc_DEFINE_NONTYPE_REPLACETYPES_(bool);
Vc_DEFINE_NONTYPE_REPLACETYPES_(wchar_t);
Vc_DEFINE_NONTYPE_REPLACETYPES_(char);
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed char);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned char);
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed short);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned short);
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed int);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned int);
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed long);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long);
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed long long);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long long);
#undef Vc_DEFINE_NONTYPE_REPLACETYPES_
namespace preferred_construction_impl
{
template <typename T> T create();
template <class Type, class... Init, class = decltype(Type(create<Init>()...))>
constexpr std::integral_constant<int, 0> test(int);
template <class Type, class... Init, class = decltype(Type{create<Init>()...})>
constexpr std::integral_constant<int, 1> test(float);
template <class Type, class... Init, class T, class = decltype(Type{{create<Init>()...}})>
constexpr std::integral_constant<int, 2> test(T);
template <class Type, class... Init> constexpr std::integral_constant<int, 3> test(...);
}
template <class Type, class... Init>
constexpr inline decltype(preferred_construction_impl::test<Type, Init...>(0))
preferred_construction()
{
return {};
}
template <size_t I, typename T,
typename R = decltype(std::declval<T &>().template vc_get_<I>())>
R get_dispatcher(T &x, void * = nullptr)
{
return x.template vc_get_<I>();
}
template <size_t I, typename T,
typename R = decltype(std::declval<const T &>().template vc_get_<I>())>
R get_dispatcher(const T &x, void * = nullptr)
{
return x.template vc_get_<I>();
}
template <size_t I, typename T, typename R = decltype(std::get<I>(std::declval<T &>()))>
R get_dispatcher(T &x, int = 0)
{
return std::get<I>(x);
}
template <size_t I, typename T,
typename R = decltype(std::get<I>(std::declval<const T &>()))>
R get_dispatcher(const T &x, int = 0)
{
return std::get<I>(x);
}
template <size_t I, class T, class = void>
struct my_tuple_element : std::tuple_element<I, T> {
};
template <size_t I, class T>
struct my_tuple_element<
I, T, typename std::conditional<
true, void, decltype(std::declval<T>().template vc_get_<I>())>::type> {
using type =
typename std::decay<decltype(std::declval<T>().template vc_get_<I>())>::type;
};
template <class... Ts> struct homogeneous_sizeof;
template <class T, class = void> struct homogeneous_sizeof_one;
template <class T>
struct homogeneous_sizeof_one<T,
typename std::enable_if<std::is_arithmetic<T>::value>::type>
: std::integral_constant<size_t, sizeof(T)> {
};
template <class T0> struct homogeneous_sizeof<T0> : homogeneous_sizeof_one<T0> {
};
template <class T0, class... Ts>
struct homogeneous_sizeof<T0, Ts...>
: std::integral_constant<size_t, homogeneous_sizeof<T0>::value ==
homogeneous_sizeof<Ts...>::value
? homogeneous_sizeof<T0>::value
: 0> {
};
template <class T, size_t... Is>
std::integral_constant<
size_t, homogeneous_sizeof<typename my_tuple_element<Is, T>::type...>::value>
homogeneous_sizeof_helper(index_sequence<Is...>);
template <class T>
struct homogeneous_sizeof_one<T, typename std::enable_if<std::is_class<T>::value>::type>
: decltype(homogeneous_sizeof_helper<T>(
make_index_sequence<determine_tuple_size_<T>::value>())) {
};
template <typename Scalar, typename Base, size_t N> class Adapter : public Base
{
private:
template <std::size_t... Indexes, int X>
Adapter(Vc::index_sequence<Indexes...>, const Scalar,
std::integral_constant<int, X>)
{
static_assert(
X < 3, "Failed to construct an object of type Base. Neither via "
"parenthesis-init, brace-init, nor double-brace init appear to work.");
}
template <std::size_t... Indexes>
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
std::integral_constant<int, 2>)
: Base{{get_dispatcher<Indexes>(x_)...}}
{
}
template <std::size_t... Indexes>
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
std::integral_constant<int, 1>)
: Base{get_dispatcher<Indexes>(x_)...}
{
}
template <std::size_t... Indexes>
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
std::integral_constant<int, 0>)
: Base(get_dispatcher<Indexes>(x_)...)
{
}
template <std::size_t... Indexes>
Adapter(Vc::index_sequence<Indexes...> seq_, const Scalar &x_)
: Adapter(seq_, x_,
preferred_construction<Base, decltype(get_dispatcher<Indexes>(
std::declval<const Scalar &>()))...>())
{
}
public:
static constexpr size_t size() { return N; }
static constexpr size_t Size = N;
using base_type = Base;
using scalar_type = Scalar;
Adapter() = default;
#if defined Vc_CLANG && Vc_CLANG < 0x30700
Vc_INTRINSIC Adapter(const Adapter &x) : Base(x) {}
#else
Adapter(const Adapter &) = default;
#endif
Adapter(Adapter &&) = default;
Adapter &operator=(const Adapter &) = default;
Adapter &operator=(Adapter &&) = default;
template <typename U, size_t TupleSize = determine_tuple_size_<Scalar>::value,
typename Seq = Vc::make_index_sequence<TupleSize>,
typename = enable_if<std::is_convertible<U, Scalar>::value>>
Adapter(U &&x_)
: Adapter(Seq(), static_cast<const Scalar &>(x_))
{
}
template <class F,
class = decltype(static_cast<Scalar>(std::declval<F>()(
size_t())))>
Adapter(F &&fun);
template <typename A0, typename... Args,
typename = typename std::enable_if<
!Traits::is_index_sequence<A0>::value &&
(sizeof...(Args) > 0 || !std::is_convertible<A0, Scalar>::value)>::type>
Adapter(A0 &&arg0_, Args &&... arguments_)
: Base(std::forward<A0>(arg0_), std::forward<Args>(arguments_)...)
{
}
template <typename T,
typename = decltype(Base(std::declval<const std::initializer_list<T> &>()))>
Adapter(const std::initializer_list<T> &l_)
: Base(l_)
{
}
void *operator new(size_t size)
{
return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
}
void *operator new(size_t, void *p_) { return p_; }
void *operator new[](size_t size)
{
return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
}
void *operator new[](size_t , void *p_) { return p_; }
void operator delete(void *ptr_, size_t) { Vc::Common::free(ptr_); }
void operator delete(void *, void *) {}
void operator delete[](void *ptr_, size_t) { Vc::Common::free(ptr_); }
void operator delete[](void *, void *) {}
};
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator==(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator!=(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator<=(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator>=(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator<(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator>(
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
}
}
namespace std
{
template <typename Scalar, typename Base, size_t N>
class tuple_size<Vc::SimdizeDetail::Adapter<Scalar, Base, N>> : public tuple_size<Base>
{
};
template <size_t I, typename Scalar, typename Base, size_t N>
class tuple_element<I, Vc::SimdizeDetail::Adapter<Scalar, Base, N>>
: public tuple_element<I, Base>
{
};
template <typename S, typename T, size_t N>
class allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
: public Vc::Allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
{
public:
template <typename U> struct rebind
{
typedef std::allocator<U> other;
};
};
}
namespace Vc_VERSIONED_NAMESPACE
{
namespace SimdizeDetail
{
template <typename T> static inline T decay_workaround(const T &x) { return x; }
template <typename S, typename T, size_t N, size_t... Indexes>
inline void assign_impl(Adapter<S, T, N> &a, size_t i, const S &x,
Vc::index_sequence<Indexes...>)
{
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(x)))...> tmp(
decay_workaround(get_dispatcher<Indexes>(x))...);
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(tmp), 0)...};
if (&unused == &unused) {}
}
template <class S, class... Args>
S construct(std::integral_constant<int, 0>, Args &&... args)
{
return S(std::forward<Args>(args)...);
}
template <class S, class... Args>
S construct(std::integral_constant<int, 1>, Args &&... args)
{
return S{std::forward<Args>(args)...};
}
template <class S, class... Args>
S construct(std::integral_constant<int, 2>, Args &&... args)
{
return S{{std::forward<Args>(args)...}};
}
template <typename S, typename T, size_t N, size_t... Indexes>
inline S extract_impl(const Adapter<S, T, N> &a, size_t i, Vc::index_sequence<Indexes...>)
{
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a)[i]))...> tmp(
decay_workaround(get_dispatcher<Indexes>(a)[i])...);
return construct<S>(
preferred_construction<S, decltype(decay_workaround(
get_dispatcher<Indexes>(a)[i]))...>(),
decay_workaround(get_dispatcher<Indexes>(a)[i])...);
}
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline Adapter<S, T, N> shifted_impl(const Adapter<S, T, N> &a, int shift,
Vc::index_sequence<Indexes...>)
{
Adapter<S, T, N> r;
auto &&unused = {(get_dispatcher<Indexes>(r) = get_dispatcher<Indexes>(a).shifted(shift), 0)...};
if (&unused == &unused) {}
return r;
}
template <typename S, typename T, size_t N>
inline Adapter<S, T, N> shifted(const Adapter<S, T, N> &a, int shift)
{
return shifted_impl(a, shift, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, S &x,
Vc::index_sequence<Indexes...>)
{
const auto &a_const = a;
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(x), 0)...};
auto &&unused2 = {(get_dispatcher<Indexes>(x) = get_dispatcher<Indexes>(tmp), 0)...};
if (&unused == &unused2) {}
}
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b,
std::size_t j, Vc::index_sequence<Indexes...>)
{
const auto &a_const = a;
const auto &b_const = b;
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(b_const)[j], 0)...};
auto &&unused2 = {(get_dispatcher<Indexes>(b)[j] = get_dispatcher<Indexes>(tmp), 0)...};
if (&unused == &unused2) {}
}
template <typename S, typename T, std::size_t N>
inline void swap(Adapter<S, T, N> &a, std::size_t i, S &x)
{
swap_impl(a, i, x, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
template <typename S, typename T, std::size_t N>
inline void swap(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b, std::size_t j)
{
swap_impl(a, i, b, j, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
template <typename A> class Scalar
{
using reference = typename std::add_lvalue_reference<A>::type;
using S = typename A::scalar_type;
using IndexSeq = Vc::make_index_sequence<determine_tuple_size<S>()>;
public:
Scalar(reference aa, size_t ii) : a(aa), i(ii) {}
Scalar(const Scalar &) = delete;
Scalar(Scalar &&) = delete;
Scalar &operator=(const Scalar &) = delete;
Scalar &operator=(Scalar &&) = delete;
void operator=(const S &x) { assign_impl(a, i, x, IndexSeq()); }
operator S() const { return extract_impl(a, i, IndexSeq()); }
template <typename AA>
friend inline void swap(Scalar<AA> &&a, typename AA::scalar_type &b);
template <typename AA>
friend inline void swap(typename AA::scalar_type &b, Scalar<AA> &&a);
template <typename AA> friend inline void swap(Scalar<AA> &&a, Scalar<AA> &&b);
private:
reference a;
size_t i;
};
template <typename A> inline void swap(Scalar<A> &&a, typename A::scalar_type &b)
{
swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
}
template <typename A> inline void swap(typename A::scalar_type &b, Scalar<A> &&a)
{
swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
}
template <typename A> inline void swap(Scalar<A> &&a, Scalar<A> &&b)
{
swap_impl(a.a, a.i, b.a, b.i, typename Scalar<A>::IndexSeq());
}
template <class S, class T, size_t N, size_t... I>
inline void load_interleaved_impl(Vc::index_sequence<I...>, Adapter<S, T, N> &a,
const S *mem)
{
const InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))>
wrapper(const_cast<S *>(mem));
Vc::tie(get_dispatcher<I>(a)...) = wrapper[0];
}
template <class S, class T, size_t N, size_t... I>
inline void store_interleaved_impl(Vc::index_sequence<I...>, const Adapter<S, T, N> &a,
S *mem)
{
InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))> wrapper(
mem);
wrapper[0] = Vc::tie(get_dispatcher<I>(a)...);
}
template <typename A> class Interface
{
using reference = typename std::add_lvalue_reference<A>::type;
using IndexSeq =
Vc::make_index_sequence<determine_tuple_size<typename A::scalar_type>()>;
public:
Interface(reference aa) : a(aa) {}
Scalar<A> operator[](size_t i)
{
return {a, i};
}
typename A::scalar_type operator[](size_t i) const
{
return extract_impl(a, i, IndexSeq());
}
A shifted(int amount) const
{
return shifted_impl(a, amount, IndexSeq());
}
void load(const typename A::scalar_type *mem) { load_interleaved(*this, mem); }
void store(typename A::scalar_type *mem) { store_interleaved(*this, mem); }
private:
reference a;
};
}
template <typename S, typename T, size_t N>
inline void assign(SimdizeDetail::Adapter<S, T, N> &a, size_t i, const S &x)
{
SimdizeDetail::assign_impl(
a, i, x, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<T>()>());
}
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
Vc_INTRINSIC void assign(V &v, size_t i, typename V::EntryType x)
{
v[i] = x;
}
template <typename S, typename T, size_t N>
inline S extract(const SimdizeDetail::Adapter<S, T, N> &a, size_t i)
{
return SimdizeDetail::extract_impl(
a, i, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<S>()>());
}
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
Vc_INTRINSIC typename V::EntryType extract(const V &v, size_t i)
{
return v[i];
}
template <class S, class T, size_t N>
inline void load_interleaved(SimdizeDetail::Adapter<S, T, N> &a, const S *mem)
{
if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
Common::unrolled_loop<std::size_t, 0, N>(
[&](std::size_t i) { assign(a, i, mem[i]); });
} else {
constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
SimdizeDetail::load_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
mem);
}
}
template <
class V, class T,
class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
Vc_INTRINSIC void load_interleaved(V &a, const T *mem)
{
a.load(mem, Vc::Unaligned);
}
template <class S, class T, size_t N>
inline void store_interleaved(const SimdizeDetail::Adapter<S, T, N> &a, S *mem)
{
if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
Common::unrolled_loop<std::size_t, 0, N>(
[&](std::size_t i) { mem[i] = extract(a, i); });
} else {
constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
SimdizeDetail::store_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
mem);
}
}
template <
class V, class T,
class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
Vc_INTRINSIC void store_interleaved(const V &a, T *mem)
{
a.store(mem, Vc::Unaligned);
}
template <typename S, typename T, size_t N>
SimdizeDetail::Interface<SimdizeDetail::Adapter<S, T, N>> decorate(
SimdizeDetail::Adapter<S, T, N> &a)
{
return {a};
}
template <typename S, typename T, size_t N>
const SimdizeDetail::Interface<const SimdizeDetail::Adapter<S, T, N>> decorate(
const SimdizeDetail::Adapter<S, T, N> &a)
{
return {a};
}
template <class V, class = typename std::enable_if<
Traits::is_simd_vector<typename std::decay<V>::type>::value>>
V &&decorate(V &&v)
{
return std::forward<V>(v);
}
namespace SimdizeDetail
{
template <typename Scalar, typename Base, size_t N>
template <class F, class>
Adapter<Scalar, Base, N>::Adapter(F &&fun)
{
for (size_t i = 0; i < N; ++i) {
Vc::assign(*this, i, fun(i));
}
}
namespace IteratorDetails
{
enum class Mutable { Yes, No };
template <typename It, typename V, size_t I, size_t End>
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I == End), It>)
{
return {};
}
template <typename It, typename V, size_t I, size_t End>
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I < End), It> it)
{
V r = fromIteratorImpl<It, V, I + 1, End>(it);
Traits::decay<decltype(get_dispatcher<I>(r))> tmp;
for (size_t j = 0; j < V::size(); ++j, ++it) {
tmp[j] = get_dispatcher<I>(*it);
}
get_dispatcher<I>(r) = tmp;
return r;
}
template <typename It, typename V>
Vc_INTRINSIC V fromIterator(enable_if<!Traits::is_simd_vector<V>::value, const It &> it)
{
return fromIteratorImpl<It, V, 0, determine_tuple_size<V>()>(it);
}
template <typename It, typename V>
Vc_INTRINSIC V fromIterator(
enable_if<
Traits::is_simd_vector<V>::value && Traits::has_contiguous_storage<It>::value, It>
it)
{
Vc_ASSERT(&*it + 1 == &*(it + 1));
return V(&*it, Vc::Unaligned);
}
template <typename It, typename V>
Vc_INTRINSIC V fromIterator(enable_if<Traits::is_simd_vector<V>::value &&
!Traits::has_contiguous_storage<It>::value,
It>
it)
{
V r;
for (size_t j = 0; j < V::size(); ++j, ++it) {
r[j] = *it;
}
return r;
}
template <typename T, typename value_vector, Mutable> class Pointer;
template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::Yes>
{
static constexpr auto Size = value_vector::size();
public:
value_vector *operator->() { return &data; }
Pointer() = delete;
Pointer(const Pointer &) = delete;
Pointer &operator=(const Pointer &) = delete;
Pointer &operator=(Pointer &&) = delete;
Pointer(Pointer &&) = default;
~Pointer()
{
for (size_t i = 0; i < Size; ++i, ++begin_iterator) {
*begin_iterator = extract(data, i);
}
}
Pointer(const T &it) : data(fromIterator<T, value_vector>(it)), begin_iterator(it) {}
private:
value_vector data;
T begin_iterator;
};
template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::No>
{
static constexpr auto Size = value_vector::size();
public:
const value_vector *operator->() const { return &data; }
Pointer() = delete;
Pointer(const Pointer &) = delete;
Pointer &operator=(const Pointer &) = delete;
Pointer &operator=(Pointer &&) = delete;
Pointer(Pointer &&) = default;
Pointer(const T &it) : data(fromIterator<T, value_vector>(it)) {}
private:
value_vector data;
};
template <typename T, typename value_vector, Mutable M> class Reference;
template <typename T, typename value_vector>
class Reference<T, value_vector, Mutable::Yes> : public value_vector
{
static constexpr auto Size = value_vector::size();
using reference = typename std::add_lvalue_reference<T>::type;
reference scalar_it;
public:
Reference(reference first_it)
: value_vector(fromIterator<T, value_vector>(first_it)), scalar_it(first_it)
{
}
Reference(const Reference &) = delete;
Reference(Reference &&) = default;
Reference &operator=(const Reference &) = delete;
Reference &operator=(Reference &&) = delete;
void operator=(const value_vector &x)
{
static_cast<value_vector &>(*this) = x;
auto it = scalar_it;
for (size_t i = 0; i < Size; ++i, ++it) {
*it = extract(x, i);
}
}
};
#define Vc_OP(op_) \
template <typename T0, typename V0, typename T1, typename V1> \
decltype(std::declval<const V0 &>() op_ std::declval<const V1 &>()) operator op_( \
const Reference<T0, V0, Mutable::Yes> &x, \
const Reference<T1, V1, Mutable::Yes> &y) \
{ \
return static_cast<const V0 &>(x) op_ static_cast<const V1 &>(y); \
}
Vc_ALL_COMPARES(Vc_OP);
Vc_ALL_ARITHMETICS(Vc_OP);
Vc_ALL_BINARY(Vc_OP);
Vc_ALL_LOGICAL(Vc_OP);
Vc_ALL_SHIFTS(Vc_OP);
#undef Vc_OP
template <typename T, typename value_vector>
class Reference<T, value_vector, Mutable::No> : public value_vector
{
static constexpr auto Size = value_vector::size();
public:
Reference(const T &it) : value_vector(fromIterator<T, value_vector>(it)) {}
Reference(const Reference &) = delete;
Reference(Reference &&) = default;
Reference &operator=(const Reference &) = delete;
Reference &operator=(Reference &&) = delete;
void operator=(const value_vector &x) = delete;
};
template <typename T, size_t N,
IteratorDetails::Mutable M =
(Traits::is_output_iterator<T>::value ? Mutable::Yes : Mutable::No),
typename V = simdize<typename std::iterator_traits<T>::value_type, N>,
size_t Size = V::Size,
typename = typename std::iterator_traits<T>::iterator_category>
class Iterator;
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size_>
class Iterator<T, N, M, V, Size_, std::forward_iterator_tag>
: public std::iterator<typename std::iterator_traits<T>::iterator_category, V,
typename std::iterator_traits<T>::difference_type,
IteratorDetails::Pointer<T, V, M>,
IteratorDetails::Reference<T, V, M>>
{
public:
using pointer = IteratorDetails::Pointer<T, V, M>;
using reference = IteratorDetails::Reference<T, V, M>;
using const_pointer = IteratorDetails::Pointer<T, V, IteratorDetails::Mutable::No>;
using const_reference =
IteratorDetails::Reference<T, V, IteratorDetails::Mutable::No>;
static constexpr std::size_t size() { return Size_; }
static constexpr std::size_t Size = Size_;
Iterator() = default;
Iterator(const T &x) : scalar_it(x) {}
Iterator(T &&x) : scalar_it(std::move(x)) {}
Iterator &operator=(const T &x)
{
scalar_it = x;
return *this;
}
Iterator &operator=(T &&x)
{
scalar_it = std::move(x);
return *this;
}
Iterator(const Iterator &) = default;
Iterator(Iterator &&) = default;
Iterator &operator=(const Iterator &) = default;
Iterator &operator=(Iterator &&) = default;
Iterator &operator++()
{
std::advance(scalar_it, Size);
return *this;
}
Iterator operator++(int)
{
Iterator copy(*this);
operator++();
return copy;
}
bool operator==(const Iterator &rhs) const
{
#ifndef NDEBUG
if (scalar_it == rhs.scalar_it) {
return true;
} else {
T it(scalar_it);
for (size_t i = 1; i < Size; ++i) {
Vc_ASSERT((++it != rhs.scalar_it));
}
return false;
}
#else
return scalar_it == rhs.scalar_it;
#endif
}
bool operator!=(const Iterator &rhs) const
{
return !operator==(rhs);
}
pointer operator->() { return scalar_it; }
reference operator*() { return scalar_it; }
const_pointer operator->() const { return scalar_it; }
const_reference operator*() const { return scalar_it; }
operator const T &() const { return scalar_it; }
protected:
T scalar_it;
};
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
class Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
: public Iterator<T, N, M, V, Size, std::forward_iterator_tag>
{
using Base = Iterator<T, N, M, V, Size, std::forward_iterator_tag>;
protected:
using Base::scalar_it;
public:
using pointer = typename Base::pointer;
using reference = typename Base::reference;
using const_pointer = typename Base::const_pointer;
using const_reference = typename Base::const_reference;
using Iterator<T, N, M, V, Size,
std::forward_iterator_tag>::Iterator;
Iterator &operator--()
{
std::advance(scalar_it, -Size);
return *this;
}
Iterator operator--(int)
{
Iterator copy(*this);
operator--();
return copy;
}
};
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
class Iterator<T, N, M, V, Size, std::random_access_iterator_tag>
: public Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
{
using Base = Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>;
protected:
using Base::scalar_it;
public:
using pointer = typename Base::pointer;
using reference = typename Base::reference;
using const_pointer = typename Base::const_pointer;
using const_reference = typename Base::const_reference;
using difference_type = typename std::iterator_traits<T>::difference_type;
using Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>::
Iterator;
Iterator &operator+=(difference_type n)
{
scalar_it += n * difference_type(Size);
return *this;
}
Iterator operator+(difference_type n) const { return Iterator(*this) += n; }
Iterator &operator-=(difference_type n)
{
scalar_it -= n * difference_type(Size);
return *this;
}
Iterator operator-(difference_type n) const { return Iterator(*this) -= n; }
difference_type operator-(const Iterator &rhs) const
{
constexpr difference_type n = Size;
Vc_ASSERT((scalar_it - rhs.scalar_it) % n ==
0);
return (scalar_it - rhs.scalar_it) / n;
}
bool operator<(const Iterator &rhs) const
{
return rhs.scalar_it - scalar_it >= difference_type(Size);
}
bool operator>(const Iterator &rhs) const
{
return scalar_it - rhs.scalar_it >= difference_type(Size);
}
bool operator<=(const Iterator &rhs) const
{
return rhs.scalar_it - scalar_it >= difference_type(Size) - 1;
}
bool operator>=(const Iterator &rhs) const
{
return scalar_it - rhs.scalar_it >= difference_type(Size) - 1;
}
reference operator[](difference_type i) { return *(*this + i); }
const_reference operator[](difference_type i) const { return *(*this + i); }
};
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
Iterator<T, N, M, V, Size, std::random_access_iterator_tag> operator+(
typename Iterator<T, N, M, V, Size, std::random_access_iterator_tag>::difference_type
n,
const Iterator<T, N, M, V, Size, std::random_access_iterator_tag> &i)
{
return i + n;
}
}
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::ForwardIterator>
{
using type = IteratorDetails::Iterator<T, N>;
};
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::BidirectionalIterator>
{
using type = IteratorDetails::Iterator<T, N>;
};
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::RandomAccessIterator>
{
using type = IteratorDetails::Iterator<T, N>;
};
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
std::size_t Offset>
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
conditional_assign(Adapter<S, T, N> &, const M &, const U &)
{
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
std::size_t Offset = 0>
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
conditional_assign(Adapter<S, T, N> &lhs, const M &mask, const U &rhs)
{
using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
using M2 = typename V::mask_type;
conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask), get_dispatcher<Offset>(rhs));
conditional_assign<Op, S, T, N, M, U, Offset + 1>(lhs, mask, rhs);
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
std::size_t Offset>
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
conditional_assign(Adapter<S, T, N> &, const M &)
{
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
std::size_t Offset = 0>
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
conditional_assign(Adapter<S, T, N> &lhs, const M &mask)
{
using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
using M2 = typename V::mask_type;
conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask));
conditional_assign<Op, S, T, N, M, Offset + 1>(lhs, mask);
}
}
template <typename T, size_t N = 0, typename MT = void>
using simdize = SimdizeDetail::simdize<T, N, MT>;
#define Vc_SIMDIZE_INTERFACE(MEMBERS_) \
template <std::size_t N_> \
inline auto vc_get_()->decltype(std::get<N_>(std::tie MEMBERS_)) \
{ \
return std::get<N_>(std::tie MEMBERS_); \
} \
template <std::size_t N_> \
inline auto vc_get_() const->decltype(std::get<N_>(std::tie MEMBERS_)) \
{ \
return std::get<N_>(std::tie MEMBERS_); \
} \
enum : std::size_t { \
tuple_size = std::tuple_size<decltype(std::make_tuple MEMBERS_)>::value \
}
}
namespace std
{
using Vc::SimdizeDetail::swap;
}
#endif
namespace Vc_VERSIONED_NAMESPACE
{
#ifdef DOXYGEN
template <class InputIt, class UnaryFunction>
UnaryFunction simd_for_each(InputIt first, InputIt last, UnaryFunction f);
#else
template <class InputIt, class UnaryFunction,
class ValueType = typename std::iterator_traits<InputIt>::value_type>
inline enable_if<
Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
UnaryFunction>
simd_for_each(InputIt first, InputIt last, UnaryFunction f)
{
typedef simdize<ValueType> V;
typedef simdize<ValueType, 1> V1;
const auto lastV = last - V::Size + 1;
for (; first < lastV; first += V::Size) {
V tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
}
for (; first != last; ++first) {
V1 tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
}
return f;
}
template <typename InputIt, typename UnaryFunction,
class ValueType = typename std::iterator_traits<InputIt>::value_type>
inline enable_if<
!Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
UnaryFunction>
simd_for_each(InputIt first, InputIt last, UnaryFunction f)
{
typedef simdize<ValueType> V;
typedef simdize<ValueType, 1> V1;
const auto lastV = last - V::size() + 1;
for (; first < lastV; first += V::size()) {
V tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
store_interleaved(tmp, std::addressof(*first));
}
for (; first != last; ++first) {
V1 tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
store_interleaved(tmp, std::addressof(*first));
}
return f;
}
#endif
template <typename InputIt, typename UnaryFunction,
class ValueType = typename std::iterator_traits<InputIt>::value_type>
inline enable_if<
Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
UnaryFunction>
simd_for_each_n(InputIt first, std::size_t count, UnaryFunction f)
{
typename std::make_signed<size_t>::type len = count;
typedef simdize<ValueType> V;
typedef simdize<ValueType, 1> V1;
for (; len >= int(V::size()); len -= V::Size, first += V::Size) {
V tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
}
for (; len != 0; --len, ++first) {
V1 tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
}
return f;
}
template <typename InputIt, typename UnaryFunction,
class ValueType = typename std::iterator_traits<InputIt>::value_type>
inline enable_if<
!Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
UnaryFunction>
simd_for_each_n(InputIt first, std::size_t count, UnaryFunction f)
{
typename std::make_signed<size_t>::type len = count;
typedef simdize<ValueType> V;
typedef simdize<ValueType, 1> V1;
for (; len >= int(V::size()); len -= V::Size, first += V::Size) {
V tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
store_interleaved(tmp, std::addressof(*first));
}
for (; len != 0; --len, ++first) {
V1 tmp;
load_interleaved(tmp, std::addressof(*first));
f(tmp);
store_interleaved(tmp, std::addressof(*first));
}
return f;
}
}
#endif