forked from openkylin/vc
1630 lines
56 KiB
Plaintext
1630 lines
56 KiB
Plaintext
|
#ifndef VC_COMMON_ALGORITHMS_H_
|
||
|
|
#define VC_COMMON_ALGORITHMS_H_
|
||
|
|
#ifndef VC_COMMON_SIMDIZE_H_
|
||
|
|
#define VC_COMMON_SIMDIZE_H_
|
||
|
|
#include <tuple>
|
||
|
|
#include <array>
|
||
|
|
#ifndef VC_ALLOCATOR_H_
|
||
|
|
#define VC_ALLOCATOR_H_
|
||
|
|
#include <new>
|
||
|
|
#include <cstddef>
|
||
|
|
#include <cstdlib>
|
||
|
|
#include <utility>
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
#define Vc_DECLARE_ALLOCATOR(Type) \
|
||
|
|
namespace std \
|
||
|
|
{ \
|
||
|
|
template <> class allocator<Type> : public ::Vc::Allocator<Type> \
|
||
|
|
{ \
|
||
|
|
public: \
|
||
|
|
template <typename U> struct rebind { \
|
||
|
|
typedef ::std::allocator<U> other; \
|
||
|
|
}; \
|
||
|
|
\
|
||
|
|
const allocator &select_on_container_copy_construction() const { return *this; } \
|
||
|
|
}; \
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
#define Vc_DECLARE_ALLOCATOR(Type) \
|
||
|
|
namespace std \
|
||
|
|
{ \
|
||
|
|
template <> class allocator<Type> : public ::Vc::Allocator<Type> \
|
||
|
|
{ \
|
||
|
|
public: \
|
||
|
|
template <typename U> struct rebind { \
|
||
|
|
typedef ::std::allocator<U> other; \
|
||
|
|
}; \
|
||
|
|
}; \
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
namespace Vc_VERSIONED_NAMESPACE
|
||
|
|
{
|
||
|
|
using std::size_t;
|
||
|
|
using std::ptrdiff_t;
|
||
|
|
template<typename T> class Allocator
|
||
|
|
{
|
||
|
|
private:
|
||
|
|
enum Constants {
|
||
|
|
#ifdef Vc_HAVE_STD_MAX_ALIGN_T
|
||
|
|
NaturalAlignment = alignof(std::max_align_t),
|
||
|
|
#elif defined(Vc_HAVE_MAX_ALIGN_T)
|
||
|
|
NaturalAlignment = alignof(::max_align_t),
|
||
|
|
#else
|
||
|
|
NaturalAlignment = sizeof(void *) > alignof(long double) ? sizeof(void *) :
|
||
|
|
(alignof(long double) > alignof(long long) ? alignof(long double) : alignof(long long)),
|
||
|
|
#endif
|
||
|
|
#if defined Vc_IMPL_AVX
|
||
|
|
SimdAlignment = 32,
|
||
|
|
#elif defined Vc_IMPL_SSE
|
||
|
|
SimdAlignment = 16,
|
||
|
|
#else
|
||
|
|
SimdAlignment = 1,
|
||
|
|
#endif
|
||
|
|
Alignment = alignof(T) > SimdAlignment ? alignof(T) : SimdAlignment,
|
||
|
|
ExtraBytes = Alignment > NaturalAlignment ? Alignment : 0,
|
||
|
|
AlignmentMask = Alignment - 1
|
||
|
|
};
|
||
|
|
public:
|
||
|
|
typedef size_t size_type;
|
||
|
|
typedef ptrdiff_t difference_type;
|
||
|
|
typedef T* pointer;
|
||
|
|
typedef const T* const_pointer;
|
||
|
|
typedef T& reference;
|
||
|
|
typedef const T& const_reference;
|
||
|
|
typedef T value_type;
|
||
|
|
template<typename U> struct rebind { typedef Allocator<U> other; };
|
||
|
|
Allocator() throw() { }
|
||
|
|
Allocator(const Allocator&) throw() { }
|
||
|
|
template<typename U> Allocator(const Allocator<U>&) throw() { }
|
||
|
|
pointer address(reference x) const { return &x; }
|
||
|
|
const_pointer address(const_reference x) const { return &x; }
|
||
|
|
pointer allocate(size_type n, const void* = 0)
|
||
|
|
{
|
||
|
|
if (n > this->max_size()) {
|
||
|
|
throw std::bad_alloc();
|
||
|
|
}
|
||
|
|
char *p = static_cast<char *>(::operator new(n * sizeof(T) + ExtraBytes));
|
||
|
|
if (ExtraBytes > 0) {
|
||
|
|
char *const pp = p;
|
||
|
|
p += ExtraBytes;
|
||
|
|
const char *null = 0;
|
||
|
|
p -= ((p - null) & AlignmentMask);
|
||
|
|
reinterpret_cast<char **>(p)[-1] = pp;
|
||
|
|
}
|
||
|
|
return reinterpret_cast<pointer>(p);
|
||
|
|
}
|
||
|
|
void deallocate(pointer p, size_type)
|
||
|
|
{
|
||
|
|
if (ExtraBytes > 0) {
|
||
|
|
p = reinterpret_cast<pointer *>(p)[-1];
|
||
|
|
}
|
||
|
|
::operator delete(p);
|
||
|
|
}
|
||
|
|
size_type max_size() const throw() { return size_t(-1) / sizeof(T); }
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
const Allocator &select_on_container_copy_construction() const { return *this; }
|
||
|
|
void construct(pointer p) { ::new(p) T(); }
|
||
|
|
void construct(pointer p, const T& val) { ::new(p) T(val); }
|
||
|
|
void destroy(pointer p) { p->~T(); }
|
||
|
|
#else
|
||
|
|
template<typename U, typename... Args> void construct(U* p, Args&&... args)
|
||
|
|
{
|
||
|
|
::new(p) U(std::forward<Args>(args)...);
|
||
|
|
}
|
||
|
|
template<typename U> void destroy(U* p) { p->~U(); }
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template<typename T> inline bool operator==(const Allocator<T>&, const Allocator<T>&) { return true; }
|
||
|
|
template<typename T> inline bool operator!=(const Allocator<T>&, const Allocator<T>&) { return false; }
|
||
|
|
}
|
||
|
|
namespace std
|
||
|
|
{
|
||
|
|
template<typename T, typename Abi>
|
||
|
|
class allocator<Vc::Vector<T, Abi> > : public ::Vc::Allocator<Vc::Vector<T, Abi> >
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
const allocator &select_on_container_copy_construction() const { return *this; }
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename T, typename Abi>
|
||
|
|
class allocator<Vc::Mask<T, Abi>> : public ::Vc::Allocator<Vc::Mask<T, Abi>>
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
const allocator &select_on_container_copy_construction() const { return *this; }
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename T, std::size_t N, typename V, std::size_t M>
|
||
|
|
class allocator<Vc::SimdArray<T, N, V, M>> : public ::Vc::Allocator<Vc::SimdArray<T, N, V, M>>
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
const allocator &select_on_container_copy_construction() const { return *this; }
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename T, std::size_t N, typename V, std::size_t M>
|
||
|
|
class allocator<Vc::SimdMaskArray<T, N, V, M>> : public ::Vc::Allocator<Vc::SimdMaskArray<T, N, V, M>>
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
template<typename U> struct rebind { typedef ::std::allocator<U> other; };
|
||
|
|
#ifdef Vc_MSVC
|
||
|
|
const allocator &select_on_container_copy_construction() const { return *this; }
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
#ifndef VC_COMMON_INTERLEAVEDMEMORY_H_
|
||
|
|
#define VC_COMMON_INTERLEAVEDMEMORY_H_
|
||
|
|
namespace Vc_VERSIONED_NAMESPACE
|
||
|
|
{
|
||
|
|
namespace Common
|
||
|
|
{
|
||
|
|
template<typename V, typename I, bool Readonly> struct InterleavedMemoryAccessBase
|
||
|
|
{
|
||
|
|
typedef typename std::conditional<
|
||
|
|
Readonly, typename std::add_const<typename V::EntryType>::type,
|
||
|
|
typename V::EntryType>::type T;
|
||
|
|
typedef typename V::AsArg VArg;
|
||
|
|
typedef T Ta Vc_MAY_ALIAS;
|
||
|
|
const I m_indexes;
|
||
|
|
Ta *const m_data;
|
||
|
|
Vc_ALWAYS_INLINE InterleavedMemoryAccessBase(typename I::AsArg indexes, Ta *data)
|
||
|
|
: m_indexes(indexes), m_data(data)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <typename... Vs> Vc_INTRINSIC void deinterleave(Vs &&... vs) const
|
||
|
|
{
|
||
|
|
Impl::deinterleave(m_data, m_indexes, std::forward<Vs>(vs)...);
|
||
|
|
}
|
||
|
|
protected:
|
||
|
|
using Impl = Vc::Detail::InterleaveImpl<V, V::Size, sizeof(V)>;
|
||
|
|
template <typename T, std::size_t... Indexes>
|
||
|
|
Vc_INTRINSIC void callInterleave(T &&a, index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
Impl::interleave(m_data, m_indexes, a[Indexes]...);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
template <size_t StructSize, typename V, typename I = typename V::IndexType,
|
||
|
|
bool Readonly>
|
||
|
|
struct InterleavedMemoryReadAccess : public InterleavedMemoryAccessBase<V, I, Readonly>
|
||
|
|
{
|
||
|
|
typedef InterleavedMemoryAccessBase<V, I, Readonly> Base;
|
||
|
|
typedef typename Base::Ta Ta;
|
||
|
|
Vc_ALWAYS_INLINE InterleavedMemoryReadAccess(Ta *data, typename I::AsArg indexes)
|
||
|
|
: Base(StructSize == 1u
|
||
|
|
? indexes
|
||
|
|
: StructSize == 2u
|
||
|
|
? indexes << 1
|
||
|
|
: StructSize == 4u
|
||
|
|
? indexes << 2
|
||
|
|
: StructSize == 8u
|
||
|
|
? indexes << 3
|
||
|
|
: StructSize == 16u ? indexes << 4
|
||
|
|
: indexes * I(int(StructSize)),
|
||
|
|
data)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <typename T, std::size_t... Indexes>
|
||
|
|
Vc_ALWAYS_INLINE T deinterleave_unpack(index_sequence<Indexes...>) const
|
||
|
|
{
|
||
|
|
T r;
|
||
|
|
Base::Impl::deinterleave(this->m_data, this->m_indexes, std::get<Indexes>(r)...);
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
template <typename T,
|
||
|
|
typename = enable_if<(std::is_default_constructible<T>::value &&
|
||
|
|
std::is_same<V, Traits::decay<decltype(std::get<0>(
|
||
|
|
std::declval<T &>()))>>::value)>>
|
||
|
|
Vc_ALWAYS_INLINE operator T() const
|
||
|
|
{
|
||
|
|
return deinterleave_unpack<T>(make_index_sequence<std::tuple_size<T>::value>());
|
||
|
|
}
|
||
|
|
};
|
||
|
|
template<typename I> struct CheckIndexesUnique
|
||
|
|
{
|
||
|
|
#ifdef NDEBUG
|
||
|
|
static Vc_INTRINSIC void test(const I &) {}
|
||
|
|
#else
|
||
|
|
static void test(const I &indexes)
|
||
|
|
{
|
||
|
|
const I test = indexes.sorted();
|
||
|
|
Vc_ASSERT(I::Size == 1 || (test == test.rotated(1)).isEmpty())
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template<size_t S> struct CheckIndexesUnique<SuccessiveEntries<S> >
|
||
|
|
{
|
||
|
|
static Vc_INTRINSIC void test(const SuccessiveEntries<S> &) {}
|
||
|
|
};
|
||
|
|
template <size_t StructSize, typename V, typename I = typename V::IndexType>
|
||
|
|
struct InterleavedMemoryAccess : public InterleavedMemoryReadAccess<StructSize, V, I, false>
|
||
|
|
{
|
||
|
|
typedef InterleavedMemoryAccessBase<V, I, false> Base;
|
||
|
|
typedef typename Base::Ta Ta;
|
||
|
|
Vc_ALWAYS_INLINE InterleavedMemoryAccess(Ta *data, typename I::AsArg indexes)
|
||
|
|
: InterleavedMemoryReadAccess<StructSize, V, I, false>(data, indexes)
|
||
|
|
{
|
||
|
|
CheckIndexesUnique<I>::test(indexes);
|
||
|
|
}
|
||
|
|
template <int N> Vc_ALWAYS_INLINE void operator=(VectorReferenceArray<N, V> &&rhs)
|
||
|
|
{
|
||
|
|
static_assert(N <= StructSize,
|
||
|
|
"You_are_trying_to_scatter_more_data_into_the_struct_than_it_has");
|
||
|
|
this->callInterleave(std::move(rhs), make_index_sequence<N>());
|
||
|
|
}
|
||
|
|
template <int N> Vc_ALWAYS_INLINE void operator=(VectorReferenceArray<N, const V> &&rhs)
|
||
|
|
{
|
||
|
|
static_assert(N <= StructSize,
|
||
|
|
"You_are_trying_to_scatter_more_data_into_the_struct_than_it_has");
|
||
|
|
this->callInterleave(std::move(rhs), make_index_sequence<N>());
|
||
|
|
}
|
||
|
|
};
|
||
|
|
template<typename S, typename V> class InterleavedMemoryWrapper
|
||
|
|
{
|
||
|
|
typedef typename std::conditional<std::is_const<S>::value,
|
||
|
|
const typename V::EntryType,
|
||
|
|
typename V::EntryType>::type T;
|
||
|
|
typedef typename V::IndexType I;
|
||
|
|
typedef typename V::AsArg VArg;
|
||
|
|
typedef const I &IndexType;
|
||
|
|
static constexpr std::size_t StructSize = sizeof(S) / sizeof(T);
|
||
|
|
using ReadAccess = InterleavedMemoryReadAccess<StructSize, V>;
|
||
|
|
using Access =
|
||
|
|
typename std::conditional<std::is_const<T>::value, ReadAccess,
|
||
|
|
InterleavedMemoryAccess<StructSize, V>>::type;
|
||
|
|
using ReadSuccessiveEntries =
|
||
|
|
InterleavedMemoryReadAccess<StructSize, V, SuccessiveEntries<StructSize>>;
|
||
|
|
using AccessSuccessiveEntries = typename std::conditional<
|
||
|
|
std::is_const<T>::value, ReadSuccessiveEntries,
|
||
|
|
InterleavedMemoryAccess<StructSize, V, SuccessiveEntries<StructSize>>>::type;
|
||
|
|
typedef T Ta Vc_MAY_ALIAS;
|
||
|
|
Ta *const m_data;
|
||
|
|
static_assert(StructSize * sizeof(T) == sizeof(S),
|
||
|
|
"InterleavedMemoryAccess_does_not_support_packed_structs");
|
||
|
|
public:
|
||
|
|
Vc_ALWAYS_INLINE InterleavedMemoryWrapper(S *s)
|
||
|
|
: m_data(reinterpret_cast<Ta *>(s))
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <typename IT>
|
||
|
|
Vc_ALWAYS_INLINE enable_if<!std::is_convertible<IT, size_t>::value &&
|
||
|
|
std::is_convertible<IT, IndexType>::value &&
|
||
|
|
!std::is_const<S>::value,
|
||
|
|
Access>
|
||
|
|
operator[](IT indexes)
|
||
|
|
{
|
||
|
|
return Access(m_data, indexes);
|
||
|
|
}
|
||
|
|
Vc_ALWAYS_INLINE ReadAccess operator[](IndexType indexes) const
|
||
|
|
{
|
||
|
|
return ReadAccess(m_data, indexes);
|
||
|
|
}
|
||
|
|
Vc_ALWAYS_INLINE ReadAccess gather(IndexType indexes) const { return operator[](indexes); }
|
||
|
|
Vc_ALWAYS_INLINE ReadSuccessiveEntries operator[](size_t first) const
|
||
|
|
{
|
||
|
|
return ReadSuccessiveEntries(m_data, first);
|
||
|
|
}
|
||
|
|
Vc_ALWAYS_INLINE AccessSuccessiveEntries operator[](size_t first)
|
||
|
|
{
|
||
|
|
return AccessSuccessiveEntries(m_data, first);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
}
|
||
|
|
using Common::InterleavedMemoryWrapper;
|
||
|
|
template <typename V, typename S>
|
||
|
|
inline Common::InterleavedMemoryWrapper<S, V> make_interleave_wrapper(S *s)
|
||
|
|
{
|
||
|
|
return Common::InterleavedMemoryWrapper<S, V>(s);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
namespace Vc_VERSIONED_NAMESPACE
|
||
|
|
{
|
||
|
|
namespace SimdizeDetail
|
||
|
|
{
|
||
|
|
using std::is_same;
|
||
|
|
using std::is_base_of;
|
||
|
|
using std::false_type;
|
||
|
|
using std::true_type;
|
||
|
|
using std::iterator_traits;
|
||
|
|
using std::conditional;
|
||
|
|
using std::size_t;
|
||
|
|
template <typename... Ts> struct Typelist;
|
||
|
|
enum class Category {
|
||
|
|
None,
|
||
|
|
ArithmeticVectorizable,
|
||
|
|
InputIterator,
|
||
|
|
OutputIterator,
|
||
|
|
ForwardIterator,
|
||
|
|
BidirectionalIterator,
|
||
|
|
RandomAccessIterator,
|
||
|
|
ClassTemplate
|
||
|
|
};
|
||
|
|
template <typename T, typename ItCat = typename T::iterator_category>
|
||
|
|
constexpr Category iteratorCategories(int, ItCat * = nullptr)
|
||
|
|
{
|
||
|
|
return is_base_of<std::random_access_iterator_tag, ItCat>::value
|
||
|
|
? Category::RandomAccessIterator
|
||
|
|
: is_base_of<std::bidirectional_iterator_tag, ItCat>::value
|
||
|
|
? Category::BidirectionalIterator
|
||
|
|
: is_base_of<std::forward_iterator_tag, ItCat>::value
|
||
|
|
? Category::ForwardIterator
|
||
|
|
: is_base_of<std::output_iterator_tag, ItCat>::value
|
||
|
|
? Category::OutputIterator
|
||
|
|
: is_base_of<std::input_iterator_tag, ItCat>::value
|
||
|
|
? Category::InputIterator
|
||
|
|
: Category::None;
|
||
|
|
}
|
||
|
|
template <typename T>
|
||
|
|
constexpr enable_if<std::is_pointer<T>::value, Category> iteratorCategories(float)
|
||
|
|
{
|
||
|
|
return Category::RandomAccessIterator;
|
||
|
|
}
|
||
|
|
template <typename T> constexpr Category iteratorCategories(...)
|
||
|
|
{
|
||
|
|
return Category::None;
|
||
|
|
}
|
||
|
|
template <typename T> struct is_class_template : public false_type
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <template <typename...> class C, typename... Ts>
|
||
|
|
struct is_class_template<C<Ts...>> : public true_type
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <typename T> constexpr Category typeCategory()
|
||
|
|
{
|
||
|
|
return (is_same<T, bool>::value || is_same<T, short>::value ||
|
||
|
|
is_same<T, unsigned short>::value || is_same<T, int>::value ||
|
||
|
|
is_same<T, unsigned int>::value || is_same<T, float>::value ||
|
||
|
|
is_same<T, double>::value)
|
||
|
|
? Category::ArithmeticVectorizable
|
||
|
|
: iteratorCategories<T>(int()) != Category::None
|
||
|
|
? iteratorCategories<T>(int())
|
||
|
|
: is_class_template<T>::value ? Category::ClassTemplate
|
||
|
|
: Category::None;
|
||
|
|
}
|
||
|
|
template <typename T, size_t TupleSize = std::tuple_size<T>::value>
|
||
|
|
constexpr size_t determine_tuple_size()
|
||
|
|
{
|
||
|
|
return TupleSize;
|
||
|
|
}
|
||
|
|
template <typename T, size_t TupleSize = T::tuple_size>
|
||
|
|
constexpr size_t determine_tuple_size(size_t = T::tuple_size)
|
||
|
|
{
|
||
|
|
return TupleSize;
|
||
|
|
}
|
||
|
|
template <typename T> struct determine_tuple_size_
|
||
|
|
: public std::integral_constant<size_t, determine_tuple_size<T>()>
|
||
|
|
{};
|
||
|
|
namespace
|
||
|
|
{
|
||
|
|
template <typename T> struct The_simdization_for_the_requested_type_is_not_implemented;
|
||
|
|
}
|
||
|
|
template <typename T, size_t N, typename MT, Category = typeCategory<T>()>
|
||
|
|
struct ReplaceTypes : public The_simdization_for_the_requested_type_is_not_implemented<T>
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, typename MT> struct ReplaceTypes<T, N, MT, Category::None>
|
||
|
|
{
|
||
|
|
typedef T type;
|
||
|
|
};
|
||
|
|
template <typename T, size_t N = 0, typename MT = void>
|
||
|
|
using simdize = typename SimdizeDetail::ReplaceTypes<T, N, MT>::type;
|
||
|
|
template <class T, size_t N,
|
||
|
|
class Best = typename Common::select_best_vector_type<T, N>::type>
|
||
|
|
using deduce_vector_t =
|
||
|
|
typename std::conditional<Best::size() == N, Best, SimdArray<T, N>>::type;
|
||
|
|
template <typename T, size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<T, N, MT, Category::ArithmeticVectorizable>
|
||
|
|
: public conditional<N == 0, Vector<T>, deduce_vector_t<T, N>> {
|
||
|
|
};
|
||
|
|
template <size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<bool, N, MT, Category::ArithmeticVectorizable>
|
||
|
|
: public std::enable_if<true, typename ReplaceTypes<MT, N, MT>::type::mask_type> {
|
||
|
|
};
|
||
|
|
template <size_t N>
|
||
|
|
struct ReplaceTypes<bool, N, void, Category::ArithmeticVectorizable>
|
||
|
|
: public ReplaceTypes<bool, N, float, Category::ArithmeticVectorizable>
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <size_t N, typename MT, typename Replaced, typename... Remaining>
|
||
|
|
struct SubstituteOneByOne;
|
||
|
|
template <size_t N, typename MT, typename... Replaced, typename T,
|
||
|
|
typename... Remaining>
|
||
|
|
struct SubstituteOneByOne<N, MT, Typelist<Replaced...>, T, Remaining...>
|
||
|
|
{
|
||
|
|
private:
|
||
|
|
template <typename U, size_t M = U::Size>
|
||
|
|
static std::integral_constant<size_t, M> size_or_0(int);
|
||
|
|
template <typename U> static std::integral_constant<size_t, 0> size_or_0(...);
|
||
|
|
using V = simdize<T, N, MT>;
|
||
|
|
static constexpr auto NewN = N != 0 ? N : decltype(size_or_0<V>(int()))::value;
|
||
|
|
typedef conditional_t<(N != NewN && is_same<MT, void>::value),
|
||
|
|
conditional_t<is_same<T, bool>::value, float, T>, MT> NewMT;
|
||
|
|
public:
|
||
|
|
using type = typename SubstituteOneByOne<NewN, NewMT, Typelist<Replaced..., V>,
|
||
|
|
Remaining...>::type;
|
||
|
|
};
|
||
|
|
template <size_t Size, typename... Replaced> struct SubstitutedBase;
|
||
|
|
template <typename Replaced> struct SubstitutedBase<1, Replaced> {
|
||
|
|
template <typename ValueT, template <typename, ValueT...> class C, ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced, Values...>;
|
||
|
|
};
|
||
|
|
template <typename R0, typename R1> struct SubstitutedBase<2, R0, R1>
|
||
|
|
{
|
||
|
|
template <typename ValueT, template <typename, typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<R0, R1, Values...>;
|
||
|
|
};
|
||
|
|
template <typename R0, typename R1, typename R2> struct SubstitutedBase<3, R0, R1, R2>
|
||
|
|
{
|
||
|
|
template <typename ValueT, template <typename, typename, typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<R0, R1, R2, Values...>;
|
||
|
|
};
|
||
|
|
#if defined Vc_ICC || defined Vc_MSVC
|
||
|
|
#define Vc_VALUE_PACK_EXPANSION_IS_BROKEN 1
|
||
|
|
#endif
|
||
|
|
template <typename... Replaced> struct SubstitutedBase<4, Replaced...> {
|
||
|
|
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
template <typename ValueT,
|
||
|
|
template <typename, typename, typename, typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced..., Values...>;
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename... Replaced> struct SubstitutedBase<5, Replaced...> {
|
||
|
|
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
template <typename ValueT, template <typename, typename, typename, typename, typename,
|
||
|
|
ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced..., Values...>;
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename... Replaced> struct SubstitutedBase<6, Replaced...> {
|
||
|
|
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
template <typename ValueT, template <typename, typename, typename, typename, typename,
|
||
|
|
typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced..., Values...>;
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename... Replaced> struct SubstitutedBase<7, Replaced...> {
|
||
|
|
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
template <typename ValueT, template <typename, typename, typename, typename, typename,
|
||
|
|
typename, typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced..., Values...>;
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <typename... Replaced> struct SubstitutedBase<8, Replaced...> {
|
||
|
|
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
template <typename ValueT, template <typename, typename, typename, typename, typename,
|
||
|
|
typename, typename, typename, ValueT...> class C,
|
||
|
|
ValueT... Values>
|
||
|
|
using SubstitutedWithValues = C<Replaced..., Values...>;
|
||
|
|
#endif
|
||
|
|
};
|
||
|
|
template <size_t N_, typename MT, typename Replaced0, typename... Replaced>
|
||
|
|
struct SubstituteOneByOne<N_, MT, Typelist<Replaced0, Replaced...>>
|
||
|
|
{
|
||
|
|
struct type
|
||
|
|
: public SubstitutedBase<sizeof...(Replaced) + 1, Replaced0, Replaced...> {
|
||
|
|
static constexpr auto N = N_;
|
||
|
|
template <template <typename...> class C>
|
||
|
|
using Substituted = C<Replaced0, Replaced...>;
|
||
|
|
};
|
||
|
|
};
|
||
|
|
template <typename Scalar, typename Base, size_t N> class Adapter;
|
||
|
|
template <template <typename...> class C, typename... Ts, size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<C<Ts...>, N, MT, Category::ClassTemplate>
|
||
|
|
{
|
||
|
|
using SubstitutionResult =
|
||
|
|
typename SubstituteOneByOne<N, MT, Typelist<>, Ts...>::type;
|
||
|
|
using Vectorized = typename SubstitutionResult::template Substituted<C>;
|
||
|
|
using type = conditional_t<is_same<C<Ts...>, Vectorized>::value, C<Ts...>,
|
||
|
|
Adapter<C<Ts...>, Vectorized, SubstitutionResult::N>>;
|
||
|
|
};
|
||
|
|
#ifdef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
|
||
|
|
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_) \
|
||
|
|
template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, ValueType_...> class C, typename T0, \
|
||
|
|
typename T1, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, ValueType_...> \
|
||
|
|
class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, typename, \
|
||
|
|
ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
typename T5, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, typename, \
|
||
|
|
typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
typename T5, typename T6, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, ValueType_> class C, typename T0, ValueType_ Value0, \
|
||
|
|
size_t N, typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, Value0>, N, MT, Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
|
||
|
|
Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t<is_same<C<T0, Value0>, Substituted>::value, C<T0, Value0>, \
|
||
|
|
Adapter<C<T0, Value0>, Substituted, NN>> type; \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, ValueType_> class C, typename T0, \
|
||
|
|
typename T1, ValueType_ Value0, size_t N, typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, T1, Value0>, N, MT, Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
|
||
|
|
Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t<is_same<C<T0, T1, Value0>, Substituted>::value, \
|
||
|
|
C<T0, T1, Value0>, \
|
||
|
|
Adapter<C<T0, T1, Value0>, Substituted, NN>> type; \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, ValueType_> class C, typename T0, \
|
||
|
|
typename T1, typename T2, ValueType_ Value0, size_t N, typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, T1, T2, Value0>, N, MT, Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0> \
|
||
|
|
Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t<is_same<C<T0, T1, T2, Value0>, Substituted>::value, \
|
||
|
|
C<T0, T1, T2, Value0>, \
|
||
|
|
Adapter<C<T0, T1, T2, Value0>, Substituted, NN>> type; \
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_) \
|
||
|
|
template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, ValueType_...> class C, typename T0, \
|
||
|
|
typename T1, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, ValueType_ Value0, \
|
||
|
|
ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, ValueType_...> \
|
||
|
|
class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, typename, \
|
||
|
|
ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
typename T5, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, typename, typename, typename, \
|
||
|
|
typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, typename T3, typename T4, \
|
||
|
|
typename T5, typename T6, ValueType_ Value0, ValueType_... Values> \
|
||
|
|
struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>> \
|
||
|
|
: public true_type { \
|
||
|
|
}; \
|
||
|
|
template <template <typename, ValueType_...> class C, typename T0, \
|
||
|
|
ValueType_ Value0, ValueType_... Values, size_t N, typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, Value0, Values...>, N, MT, Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
|
||
|
|
Values...> Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t<is_same<C<T0, Value0, Values...>, Substituted>::value, \
|
||
|
|
C<T0, Value0, Values...>, \
|
||
|
|
Adapter<C<T0, Value0, Values...>, Substituted, NN>> type; \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, ValueType_...> class C, typename T0, \
|
||
|
|
typename T1, ValueType_ Value0, ValueType_... Values, size_t N, \
|
||
|
|
typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, T1, Value0, Values...>, N, MT, Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
|
||
|
|
Values...> Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t<is_same<C<T0, T1, Value0, Values...>, Substituted>::value, \
|
||
|
|
C<T0, T1, Value0, Values...>, \
|
||
|
|
Adapter<C<T0, T1, Value0, Values...>, Substituted, NN>> \
|
||
|
|
type; \
|
||
|
|
}; \
|
||
|
|
template <template <typename, typename, typename, ValueType_...> class C, \
|
||
|
|
typename T0, typename T1, typename T2, ValueType_ Value0, \
|
||
|
|
ValueType_... Values, size_t N, typename MT> \
|
||
|
|
struct ReplaceTypes<C<T0, T1, T2, Value0, Values...>, N, MT, \
|
||
|
|
Category::ClassTemplate> { \
|
||
|
|
typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp; \
|
||
|
|
typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0, \
|
||
|
|
Values...> Substituted; \
|
||
|
|
static constexpr auto NN = tmp::N; \
|
||
|
|
typedef conditional_t< \
|
||
|
|
is_same<C<T0, T1, T2, Value0, Values...>, Substituted>::value, \
|
||
|
|
C<T0, T1, T2, Value0, Values...>, \
|
||
|
|
Adapter<C<T0, T1, T2, Value0, Values...>, Substituted, NN>> type; \
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(bool);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(wchar_t);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(char);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed char);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned char);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed short);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned short);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed int);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned int);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed long);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_( signed long long);
|
||
|
|
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long long);
|
||
|
|
#undef Vc_DEFINE_NONTYPE_REPLACETYPES_
|
||
|
|
namespace preferred_construction_impl
|
||
|
|
{
|
||
|
|
template <typename T> T create();
|
||
|
|
template <class Type, class... Init, class = decltype(Type(create<Init>()...))>
|
||
|
|
constexpr std::integral_constant<int, 0> test(int);
|
||
|
|
template <class Type, class... Init, class = decltype(Type{create<Init>()...})>
|
||
|
|
constexpr std::integral_constant<int, 1> test(float);
|
||
|
|
template <class Type, class... Init, class T, class = decltype(Type{{create<Init>()...}})>
|
||
|
|
constexpr std::integral_constant<int, 2> test(T);
|
||
|
|
template <class Type, class... Init> constexpr std::integral_constant<int, 3> test(...);
|
||
|
|
}
|
||
|
|
template <class Type, class... Init>
|
||
|
|
constexpr inline decltype(preferred_construction_impl::test<Type, Init...>(0))
|
||
|
|
preferred_construction()
|
||
|
|
{
|
||
|
|
return {};
|
||
|
|
}
|
||
|
|
template <size_t I, typename T,
|
||
|
|
typename R = decltype(std::declval<T &>().template vc_get_<I>())>
|
||
|
|
R get_dispatcher(T &x, void * = nullptr)
|
||
|
|
{
|
||
|
|
return x.template vc_get_<I>();
|
||
|
|
}
|
||
|
|
template <size_t I, typename T,
|
||
|
|
typename R = decltype(std::declval<const T &>().template vc_get_<I>())>
|
||
|
|
R get_dispatcher(const T &x, void * = nullptr)
|
||
|
|
{
|
||
|
|
return x.template vc_get_<I>();
|
||
|
|
}
|
||
|
|
template <size_t I, typename T, typename R = decltype(std::get<I>(std::declval<T &>()))>
|
||
|
|
R get_dispatcher(T &x, int = 0)
|
||
|
|
{
|
||
|
|
return std::get<I>(x);
|
||
|
|
}
|
||
|
|
template <size_t I, typename T,
|
||
|
|
typename R = decltype(std::get<I>(std::declval<const T &>()))>
|
||
|
|
R get_dispatcher(const T &x, int = 0)
|
||
|
|
{
|
||
|
|
return std::get<I>(x);
|
||
|
|
}
|
||
|
|
template <size_t I, class T, class = void>
|
||
|
|
struct my_tuple_element : std::tuple_element<I, T> {
|
||
|
|
};
|
||
|
|
template <size_t I, class T>
|
||
|
|
struct my_tuple_element<
|
||
|
|
I, T, typename std::conditional<
|
||
|
|
true, void, decltype(std::declval<T>().template vc_get_<I>())>::type> {
|
||
|
|
using type =
|
||
|
|
typename std::decay<decltype(std::declval<T>().template vc_get_<I>())>::type;
|
||
|
|
};
|
||
|
|
template <class... Ts> struct homogeneous_sizeof;
|
||
|
|
template <class T, class = void> struct homogeneous_sizeof_one;
|
||
|
|
template <class T>
|
||
|
|
struct homogeneous_sizeof_one<T,
|
||
|
|
typename std::enable_if<std::is_arithmetic<T>::value>::type>
|
||
|
|
: std::integral_constant<size_t, sizeof(T)> {
|
||
|
|
};
|
||
|
|
template <class T0> struct homogeneous_sizeof<T0> : homogeneous_sizeof_one<T0> {
|
||
|
|
};
|
||
|
|
template <class T0, class... Ts>
|
||
|
|
struct homogeneous_sizeof<T0, Ts...>
|
||
|
|
: std::integral_constant<size_t, homogeneous_sizeof<T0>::value ==
|
||
|
|
homogeneous_sizeof<Ts...>::value
|
||
|
|
? homogeneous_sizeof<T0>::value
|
||
|
|
: 0> {
|
||
|
|
};
|
||
|
|
template <class T, size_t... Is>
|
||
|
|
std::integral_constant<
|
||
|
|
size_t, homogeneous_sizeof<typename my_tuple_element<Is, T>::type...>::value>
|
||
|
|
homogeneous_sizeof_helper(index_sequence<Is...>);
|
||
|
|
template <class T>
|
||
|
|
struct homogeneous_sizeof_one<T, typename std::enable_if<std::is_class<T>::value>::type>
|
||
|
|
: decltype(homogeneous_sizeof_helper<T>(
|
||
|
|
make_index_sequence<determine_tuple_size_<T>::value>())) {
|
||
|
|
};
|
||
|
|
template <typename Scalar, typename Base, size_t N> class Adapter : public Base
|
||
|
|
{
|
||
|
|
private:
|
||
|
|
template <std::size_t... Indexes, int X>
|
||
|
|
Adapter(Vc::index_sequence<Indexes...>, const Scalar,
|
||
|
|
std::integral_constant<int, X>)
|
||
|
|
{
|
||
|
|
static_assert(
|
||
|
|
X < 3, "Failed to construct an object of type Base. Neither via "
|
||
|
|
"parenthesis-init, brace-init, nor double-brace init appear to work.");
|
||
|
|
}
|
||
|
|
template <std::size_t... Indexes>
|
||
|
|
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
|
||
|
|
std::integral_constant<int, 2>)
|
||
|
|
: Base{{get_dispatcher<Indexes>(x_)...}}
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <std::size_t... Indexes>
|
||
|
|
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
|
||
|
|
std::integral_constant<int, 1>)
|
||
|
|
: Base{get_dispatcher<Indexes>(x_)...}
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <std::size_t... Indexes>
|
||
|
|
Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
|
||
|
|
std::integral_constant<int, 0>)
|
||
|
|
: Base(get_dispatcher<Indexes>(x_)...)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <std::size_t... Indexes>
|
||
|
|
Adapter(Vc::index_sequence<Indexes...> seq_, const Scalar &x_)
|
||
|
|
: Adapter(seq_, x_,
|
||
|
|
preferred_construction<Base, decltype(get_dispatcher<Indexes>(
|
||
|
|
std::declval<const Scalar &>()))...>())
|
||
|
|
{
|
||
|
|
}
|
||
|
|
public:
|
||
|
|
static constexpr size_t size() { return N; }
|
||
|
|
static constexpr size_t Size = N;
|
||
|
|
using base_type = Base;
|
||
|
|
using scalar_type = Scalar;
|
||
|
|
Adapter() = default;
|
||
|
|
#if defined Vc_CLANG && Vc_CLANG < 0x30700
|
||
|
|
Vc_INTRINSIC Adapter(const Adapter &x) : Base(x) {}
|
||
|
|
#else
|
||
|
|
Adapter(const Adapter &) = default;
|
||
|
|
#endif
|
||
|
|
Adapter(Adapter &&) = default;
|
||
|
|
Adapter &operator=(const Adapter &) = default;
|
||
|
|
Adapter &operator=(Adapter &&) = default;
|
||
|
|
template <typename U, size_t TupleSize = determine_tuple_size_<Scalar>::value,
|
||
|
|
typename Seq = Vc::make_index_sequence<TupleSize>,
|
||
|
|
typename = enable_if<std::is_convertible<U, Scalar>::value>>
|
||
|
|
Adapter(U &&x_)
|
||
|
|
: Adapter(Seq(), static_cast<const Scalar &>(x_))
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <class F,
|
||
|
|
class = decltype(static_cast<Scalar>(std::declval<F>()(
|
||
|
|
size_t())))>
|
||
|
|
Adapter(F &&fun);
|
||
|
|
template <typename A0, typename... Args,
|
||
|
|
typename = typename std::enable_if<
|
||
|
|
!Traits::is_index_sequence<A0>::value &&
|
||
|
|
(sizeof...(Args) > 0 || !std::is_convertible<A0, Scalar>::value)>::type>
|
||
|
|
Adapter(A0 &&arg0_, Args &&... arguments_)
|
||
|
|
: Base(std::forward<A0>(arg0_), std::forward<Args>(arguments_)...)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <typename T,
|
||
|
|
typename = decltype(Base(std::declval<const std::initializer_list<T> &>()))>
|
||
|
|
Adapter(const std::initializer_list<T> &l_)
|
||
|
|
: Base(l_)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
void *operator new(size_t size)
|
||
|
|
{
|
||
|
|
return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
|
||
|
|
}
|
||
|
|
void *operator new(size_t, void *p_) { return p_; }
|
||
|
|
void *operator new[](size_t size)
|
||
|
|
{
|
||
|
|
return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
|
||
|
|
}
|
||
|
|
void *operator new[](size_t , void *p_) { return p_; }
|
||
|
|
void operator delete(void *ptr_, size_t) { Vc::Common::free(ptr_); }
|
||
|
|
void operator delete(void *, void *) {}
|
||
|
|
void operator delete[](void *ptr_, size_t) { Vc::Common::free(ptr_); }
|
||
|
|
void operator delete[](void *, void *) {}
|
||
|
|
};
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator==(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator!=(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator<=(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator>=(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator<(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
|
||
|
|
inline void operator>(
|
||
|
|
const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
|
||
|
|
const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
namespace std
|
||
|
|
{
|
||
|
|
template <typename Scalar, typename Base, size_t N>
|
||
|
|
class tuple_size<Vc::SimdizeDetail::Adapter<Scalar, Base, N>> : public tuple_size<Base>
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <size_t I, typename Scalar, typename Base, size_t N>
|
||
|
|
class tuple_element<I, Vc::SimdizeDetail::Adapter<Scalar, Base, N>>
|
||
|
|
: public tuple_element<I, Base>
|
||
|
|
{
|
||
|
|
};
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
class allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
|
||
|
|
: public Vc::Allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
template <typename U> struct rebind
|
||
|
|
{
|
||
|
|
typedef std::allocator<U> other;
|
||
|
|
};
|
||
|
|
};
|
||
|
|
}
|
||
|
|
namespace Vc_VERSIONED_NAMESPACE
|
||
|
|
{
|
||
|
|
namespace SimdizeDetail
|
||
|
|
{
|
||
|
|
template <typename T> static inline T decay_workaround(const T &x) { return x; }
|
||
|
|
template <typename S, typename T, size_t N, size_t... Indexes>
|
||
|
|
inline void assign_impl(Adapter<S, T, N> &a, size_t i, const S &x,
|
||
|
|
Vc::index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(x)))...> tmp(
|
||
|
|
decay_workaround(get_dispatcher<Indexes>(x))...);
|
||
|
|
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(tmp), 0)...};
|
||
|
|
if (&unused == &unused) {}
|
||
|
|
}
|
||
|
|
template <class S, class... Args>
|
||
|
|
S construct(std::integral_constant<int, 0>, Args &&... args)
|
||
|
|
{
|
||
|
|
return S(std::forward<Args>(args)...);
|
||
|
|
}
|
||
|
|
template <class S, class... Args>
|
||
|
|
S construct(std::integral_constant<int, 1>, Args &&... args)
|
||
|
|
{
|
||
|
|
return S{std::forward<Args>(args)...};
|
||
|
|
}
|
||
|
|
template <class S, class... Args>
|
||
|
|
S construct(std::integral_constant<int, 2>, Args &&... args)
|
||
|
|
{
|
||
|
|
return S{{std::forward<Args>(args)...}};
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N, size_t... Indexes>
|
||
|
|
inline S extract_impl(const Adapter<S, T, N> &a, size_t i, Vc::index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a)[i]))...> tmp(
|
||
|
|
decay_workaround(get_dispatcher<Indexes>(a)[i])...);
|
||
|
|
return construct<S>(
|
||
|
|
preferred_construction<S, decltype(decay_workaround(
|
||
|
|
get_dispatcher<Indexes>(a)[i]))...>(),
|
||
|
|
decay_workaround(get_dispatcher<Indexes>(a)[i])...);
|
||
|
|
}
|
||
|
|
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
|
||
|
|
inline Adapter<S, T, N> shifted_impl(const Adapter<S, T, N> &a, int shift,
|
||
|
|
Vc::index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
Adapter<S, T, N> r;
|
||
|
|
auto &&unused = {(get_dispatcher<Indexes>(r) = get_dispatcher<Indexes>(a).shifted(shift), 0)...};
|
||
|
|
if (&unused == &unused) {}
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
inline Adapter<S, T, N> shifted(const Adapter<S, T, N> &a, int shift)
|
||
|
|
{
|
||
|
|
return shifted_impl(a, shift, Vc::make_index_sequence<determine_tuple_size<T>()>());
|
||
|
|
}
|
||
|
|
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
|
||
|
|
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, S &x,
|
||
|
|
Vc::index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
const auto &a_const = a;
|
||
|
|
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
|
||
|
|
tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
|
||
|
|
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(x), 0)...};
|
||
|
|
auto &&unused2 = {(get_dispatcher<Indexes>(x) = get_dispatcher<Indexes>(tmp), 0)...};
|
||
|
|
if (&unused == &unused2) {}
|
||
|
|
}
|
||
|
|
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
|
||
|
|
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b,
|
||
|
|
std::size_t j, Vc::index_sequence<Indexes...>)
|
||
|
|
{
|
||
|
|
const auto &a_const = a;
|
||
|
|
const auto &b_const = b;
|
||
|
|
const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
|
||
|
|
tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
|
||
|
|
auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(b_const)[j], 0)...};
|
||
|
|
auto &&unused2 = {(get_dispatcher<Indexes>(b)[j] = get_dispatcher<Indexes>(tmp), 0)...};
|
||
|
|
if (&unused == &unused2) {}
|
||
|
|
}
|
||
|
|
template <typename S, typename T, std::size_t N>
|
||
|
|
inline void swap(Adapter<S, T, N> &a, std::size_t i, S &x)
|
||
|
|
{
|
||
|
|
swap_impl(a, i, x, Vc::make_index_sequence<determine_tuple_size<T>()>());
|
||
|
|
}
|
||
|
|
template <typename S, typename T, std::size_t N>
|
||
|
|
inline void swap(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b, std::size_t j)
|
||
|
|
{
|
||
|
|
swap_impl(a, i, b, j, Vc::make_index_sequence<determine_tuple_size<T>()>());
|
||
|
|
}
|
||
|
|
template <typename A> class Scalar
|
||
|
|
{
|
||
|
|
using reference = typename std::add_lvalue_reference<A>::type;
|
||
|
|
using S = typename A::scalar_type;
|
||
|
|
using IndexSeq = Vc::make_index_sequence<determine_tuple_size<S>()>;
|
||
|
|
public:
|
||
|
|
Scalar(reference aa, size_t ii) : a(aa), i(ii) {}
|
||
|
|
Scalar(const Scalar &) = delete;
|
||
|
|
Scalar(Scalar &&) = delete;
|
||
|
|
Scalar &operator=(const Scalar &) = delete;
|
||
|
|
Scalar &operator=(Scalar &&) = delete;
|
||
|
|
void operator=(const S &x) { assign_impl(a, i, x, IndexSeq()); }
|
||
|
|
operator S() const { return extract_impl(a, i, IndexSeq()); }
|
||
|
|
template <typename AA>
|
||
|
|
friend inline void swap(Scalar<AA> &&a, typename AA::scalar_type &b);
|
||
|
|
template <typename AA>
|
||
|
|
friend inline void swap(typename AA::scalar_type &b, Scalar<AA> &&a);
|
||
|
|
template <typename AA> friend inline void swap(Scalar<AA> &&a, Scalar<AA> &&b);
|
||
|
|
private:
|
||
|
|
reference a;
|
||
|
|
size_t i;
|
||
|
|
};
|
||
|
|
template <typename A> inline void swap(Scalar<A> &&a, typename A::scalar_type &b)
|
||
|
|
{
|
||
|
|
swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
|
||
|
|
}
|
||
|
|
template <typename A> inline void swap(typename A::scalar_type &b, Scalar<A> &&a)
|
||
|
|
{
|
||
|
|
swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
|
||
|
|
}
|
||
|
|
template <typename A> inline void swap(Scalar<A> &&a, Scalar<A> &&b)
|
||
|
|
{
|
||
|
|
swap_impl(a.a, a.i, b.a, b.i, typename Scalar<A>::IndexSeq());
|
||
|
|
}
|
||
|
|
template <class S, class T, size_t N, size_t... I>
|
||
|
|
inline void load_interleaved_impl(Vc::index_sequence<I...>, Adapter<S, T, N> &a,
|
||
|
|
const S *mem)
|
||
|
|
{
|
||
|
|
const InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))>
|
||
|
|
wrapper(const_cast<S *>(mem));
|
||
|
|
Vc::tie(get_dispatcher<I>(a)...) = wrapper[0];
|
||
|
|
}
|
||
|
|
template <class S, class T, size_t N, size_t... I>
|
||
|
|
inline void store_interleaved_impl(Vc::index_sequence<I...>, const Adapter<S, T, N> &a,
|
||
|
|
S *mem)
|
||
|
|
{
|
||
|
|
InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))> wrapper(
|
||
|
|
mem);
|
||
|
|
wrapper[0] = Vc::tie(get_dispatcher<I>(a)...);
|
||
|
|
}
|
||
|
|
template <typename A> class Interface
|
||
|
|
{
|
||
|
|
using reference = typename std::add_lvalue_reference<A>::type;
|
||
|
|
using IndexSeq =
|
||
|
|
Vc::make_index_sequence<determine_tuple_size<typename A::scalar_type>()>;
|
||
|
|
public:
|
||
|
|
Interface(reference aa) : a(aa) {}
|
||
|
|
Scalar<A> operator[](size_t i)
|
||
|
|
{
|
||
|
|
return {a, i};
|
||
|
|
}
|
||
|
|
typename A::scalar_type operator[](size_t i) const
|
||
|
|
{
|
||
|
|
return extract_impl(a, i, IndexSeq());
|
||
|
|
}
|
||
|
|
A shifted(int amount) const
|
||
|
|
{
|
||
|
|
return shifted_impl(a, amount, IndexSeq());
|
||
|
|
}
|
||
|
|
void load(const typename A::scalar_type *mem) { load_interleaved(*this, mem); }
|
||
|
|
void store(typename A::scalar_type *mem) { store_interleaved(*this, mem); }
|
||
|
|
private:
|
||
|
|
reference a;
|
||
|
|
};
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
inline void assign(SimdizeDetail::Adapter<S, T, N> &a, size_t i, const S &x)
|
||
|
|
{
|
||
|
|
SimdizeDetail::assign_impl(
|
||
|
|
a, i, x, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<T>()>());
|
||
|
|
}
|
||
|
|
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
|
||
|
|
Vc_INTRINSIC void assign(V &v, size_t i, typename V::EntryType x)
|
||
|
|
{
|
||
|
|
v[i] = x;
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
inline S extract(const SimdizeDetail::Adapter<S, T, N> &a, size_t i)
|
||
|
|
{
|
||
|
|
return SimdizeDetail::extract_impl(
|
||
|
|
a, i, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<S>()>());
|
||
|
|
}
|
||
|
|
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
|
||
|
|
Vc_INTRINSIC typename V::EntryType extract(const V &v, size_t i)
|
||
|
|
{
|
||
|
|
return v[i];
|
||
|
|
}
|
||
|
|
template <class S, class T, size_t N>
|
||
|
|
inline void load_interleaved(SimdizeDetail::Adapter<S, T, N> &a, const S *mem)
|
||
|
|
{
|
||
|
|
if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
|
||
|
|
Common::unrolled_loop<std::size_t, 0, N>(
|
||
|
|
[&](std::size_t i) { assign(a, i, mem[i]); });
|
||
|
|
} else {
|
||
|
|
constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
|
||
|
|
SimdizeDetail::load_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
|
||
|
|
mem);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
template <
|
||
|
|
class V, class T,
|
||
|
|
class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
|
||
|
|
Vc_INTRINSIC void load_interleaved(V &a, const T *mem)
|
||
|
|
{
|
||
|
|
a.load(mem, Vc::Unaligned);
|
||
|
|
}
|
||
|
|
template <class S, class T, size_t N>
|
||
|
|
inline void store_interleaved(const SimdizeDetail::Adapter<S, T, N> &a, S *mem)
|
||
|
|
{
|
||
|
|
if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
|
||
|
|
Common::unrolled_loop<std::size_t, 0, N>(
|
||
|
|
[&](std::size_t i) { mem[i] = extract(a, i); });
|
||
|
|
} else {
|
||
|
|
constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
|
||
|
|
SimdizeDetail::store_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
|
||
|
|
mem);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
template <
|
||
|
|
class V, class T,
|
||
|
|
class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
|
||
|
|
Vc_INTRINSIC void store_interleaved(const V &a, T *mem)
|
||
|
|
{
|
||
|
|
a.store(mem, Vc::Unaligned);
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
SimdizeDetail::Interface<SimdizeDetail::Adapter<S, T, N>> decorate(
|
||
|
|
SimdizeDetail::Adapter<S, T, N> &a)
|
||
|
|
{
|
||
|
|
return {a};
|
||
|
|
}
|
||
|
|
template <typename S, typename T, size_t N>
|
||
|
|
const SimdizeDetail::Interface<const SimdizeDetail::Adapter<S, T, N>> decorate(
|
||
|
|
const SimdizeDetail::Adapter<S, T, N> &a)
|
||
|
|
{
|
||
|
|
return {a};
|
||
|
|
}
|
||
|
|
template <class V, class = typename std::enable_if<
|
||
|
|
Traits::is_simd_vector<typename std::decay<V>::type>::value>>
|
||
|
|
V &&decorate(V &&v)
|
||
|
|
{
|
||
|
|
return std::forward<V>(v);
|
||
|
|
}
|
||
|
|
namespace SimdizeDetail
|
||
|
|
{
|
||
|
|
template <typename Scalar, typename Base, size_t N>
|
||
|
|
template <class F, class>
|
||
|
|
Adapter<Scalar, Base, N>::Adapter(F &&fun)
|
||
|
|
{
|
||
|
|
for (size_t i = 0; i < N; ++i) {
|
||
|
|
Vc::assign(*this, i, fun(i));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
namespace IteratorDetails
|
||
|
|
{
|
||
|
|
enum class Mutable { Yes, No };
|
||
|
|
template <typename It, typename V, size_t I, size_t End>
|
||
|
|
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I == End), It>)
|
||
|
|
{
|
||
|
|
return {};
|
||
|
|
}
|
||
|
|
template <typename It, typename V, size_t I, size_t End>
|
||
|
|
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I < End), It> it)
|
||
|
|
{
|
||
|
|
V r = fromIteratorImpl<It, V, I + 1, End>(it);
|
||
|
|
Traits::decay<decltype(get_dispatcher<I>(r))> tmp;
|
||
|
|
for (size_t j = 0; j < V::size(); ++j, ++it) {
|
||
|
|
tmp[j] = get_dispatcher<I>(*it);
|
||
|
|
}
|
||
|
|
get_dispatcher<I>(r) = tmp;
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
template <typename It, typename V>
|
||
|
|
Vc_INTRINSIC V fromIterator(enable_if<!Traits::is_simd_vector<V>::value, const It &> it)
|
||
|
|
{
|
||
|
|
return fromIteratorImpl<It, V, 0, determine_tuple_size<V>()>(it);
|
||
|
|
}
|
||
|
|
template <typename It, typename V>
|
||
|
|
Vc_INTRINSIC V fromIterator(
|
||
|
|
enable_if<
|
||
|
|
Traits::is_simd_vector<V>::value && Traits::has_contiguous_storage<It>::value, It>
|
||
|
|
it)
|
||
|
|
{
|
||
|
|
Vc_ASSERT(&*it + 1 == &*(it + 1));
|
||
|
|
return V(&*it, Vc::Unaligned);
|
||
|
|
}
|
||
|
|
template <typename It, typename V>
|
||
|
|
Vc_INTRINSIC V fromIterator(enable_if<Traits::is_simd_vector<V>::value &&
|
||
|
|
!Traits::has_contiguous_storage<It>::value,
|
||
|
|
It>
|
||
|
|
it)
|
||
|
|
{
|
||
|
|
V r;
|
||
|
|
for (size_t j = 0; j < V::size(); ++j, ++it) {
|
||
|
|
r[j] = *it;
|
||
|
|
}
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
template <typename T, typename value_vector, Mutable> class Pointer;
|
||
|
|
template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::Yes>
|
||
|
|
{
|
||
|
|
static constexpr auto Size = value_vector::size();
|
||
|
|
public:
|
||
|
|
value_vector *operator->() { return &data; }
|
||
|
|
Pointer() = delete;
|
||
|
|
Pointer(const Pointer &) = delete;
|
||
|
|
Pointer &operator=(const Pointer &) = delete;
|
||
|
|
Pointer &operator=(Pointer &&) = delete;
|
||
|
|
Pointer(Pointer &&) = default;
|
||
|
|
~Pointer()
|
||
|
|
{
|
||
|
|
for (size_t i = 0; i < Size; ++i, ++begin_iterator) {
|
||
|
|
*begin_iterator = extract(data, i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
Pointer(const T &it) : data(fromIterator<T, value_vector>(it)), begin_iterator(it) {}
|
||
|
|
private:
|
||
|
|
value_vector data;
|
||
|
|
T begin_iterator;
|
||
|
|
};
|
||
|
|
template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::No>
|
||
|
|
{
|
||
|
|
static constexpr auto Size = value_vector::size();
|
||
|
|
public:
|
||
|
|
const value_vector *operator->() const { return &data; }
|
||
|
|
Pointer() = delete;
|
||
|
|
Pointer(const Pointer &) = delete;
|
||
|
|
Pointer &operator=(const Pointer &) = delete;
|
||
|
|
Pointer &operator=(Pointer &&) = delete;
|
||
|
|
Pointer(Pointer &&) = default;
|
||
|
|
Pointer(const T &it) : data(fromIterator<T, value_vector>(it)) {}
|
||
|
|
private:
|
||
|
|
value_vector data;
|
||
|
|
};
|
||
|
|
template <typename T, typename value_vector, Mutable M> class Reference;
|
||
|
|
template <typename T, typename value_vector>
|
||
|
|
class Reference<T, value_vector, Mutable::Yes> : public value_vector
|
||
|
|
{
|
||
|
|
static constexpr auto Size = value_vector::size();
|
||
|
|
using reference = typename std::add_lvalue_reference<T>::type;
|
||
|
|
reference scalar_it;
|
||
|
|
public:
|
||
|
|
Reference(reference first_it)
|
||
|
|
: value_vector(fromIterator<T, value_vector>(first_it)), scalar_it(first_it)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
Reference(const Reference &) = delete;
|
||
|
|
Reference(Reference &&) = default;
|
||
|
|
Reference &operator=(const Reference &) = delete;
|
||
|
|
Reference &operator=(Reference &&) = delete;
|
||
|
|
void operator=(const value_vector &x)
|
||
|
|
{
|
||
|
|
static_cast<value_vector &>(*this) = x;
|
||
|
|
auto it = scalar_it;
|
||
|
|
for (size_t i = 0; i < Size; ++i, ++it) {
|
||
|
|
*it = extract(x, i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
};
|
||
|
|
#define Vc_OP(op_) \
|
||
|
|
template <typename T0, typename V0, typename T1, typename V1> \
|
||
|
|
decltype(std::declval<const V0 &>() op_ std::declval<const V1 &>()) operator op_( \
|
||
|
|
const Reference<T0, V0, Mutable::Yes> &x, \
|
||
|
|
const Reference<T1, V1, Mutable::Yes> &y) \
|
||
|
|
{ \
|
||
|
|
return static_cast<const V0 &>(x) op_ static_cast<const V1 &>(y); \
|
||
|
|
}
|
||
|
|
Vc_ALL_COMPARES(Vc_OP);
|
||
|
|
Vc_ALL_ARITHMETICS(Vc_OP);
|
||
|
|
Vc_ALL_BINARY(Vc_OP);
|
||
|
|
Vc_ALL_LOGICAL(Vc_OP);
|
||
|
|
Vc_ALL_SHIFTS(Vc_OP);
|
||
|
|
#undef Vc_OP
|
||
|
|
template <typename T, typename value_vector>
|
||
|
|
class Reference<T, value_vector, Mutable::No> : public value_vector
|
||
|
|
{
|
||
|
|
static constexpr auto Size = value_vector::size();
|
||
|
|
public:
|
||
|
|
Reference(const T &it) : value_vector(fromIterator<T, value_vector>(it)) {}
|
||
|
|
Reference(const Reference &) = delete;
|
||
|
|
Reference(Reference &&) = default;
|
||
|
|
Reference &operator=(const Reference &) = delete;
|
||
|
|
Reference &operator=(Reference &&) = delete;
|
||
|
|
void operator=(const value_vector &x) = delete;
|
||
|
|
};
|
||
|
|
template <typename T, size_t N,
|
||
|
|
IteratorDetails::Mutable M =
|
||
|
|
(Traits::is_output_iterator<T>::value ? Mutable::Yes : Mutable::No),
|
||
|
|
typename V = simdize<typename std::iterator_traits<T>::value_type, N>,
|
||
|
|
size_t Size = V::Size,
|
||
|
|
typename = typename std::iterator_traits<T>::iterator_category>
|
||
|
|
class Iterator;
|
||
|
|
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size_>
|
||
|
|
class Iterator<T, N, M, V, Size_, std::forward_iterator_tag>
|
||
|
|
: public std::iterator<typename std::iterator_traits<T>::iterator_category, V,
|
||
|
|
typename std::iterator_traits<T>::difference_type,
|
||
|
|
IteratorDetails::Pointer<T, V, M>,
|
||
|
|
IteratorDetails::Reference<T, V, M>>
|
||
|
|
{
|
||
|
|
public:
|
||
|
|
using pointer = IteratorDetails::Pointer<T, V, M>;
|
||
|
|
using reference = IteratorDetails::Reference<T, V, M>;
|
||
|
|
using const_pointer = IteratorDetails::Pointer<T, V, IteratorDetails::Mutable::No>;
|
||
|
|
using const_reference =
|
||
|
|
IteratorDetails::Reference<T, V, IteratorDetails::Mutable::No>;
|
||
|
|
static constexpr std::size_t size() { return Size_; }
|
||
|
|
static constexpr std::size_t Size = Size_;
|
||
|
|
Iterator() = default;
|
||
|
|
Iterator(const T &x) : scalar_it(x) {}
|
||
|
|
Iterator(T &&x) : scalar_it(std::move(x)) {}
|
||
|
|
Iterator &operator=(const T &x)
|
||
|
|
{
|
||
|
|
scalar_it = x;
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator &operator=(T &&x)
|
||
|
|
{
|
||
|
|
scalar_it = std::move(x);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator(const Iterator &) = default;
|
||
|
|
Iterator(Iterator &&) = default;
|
||
|
|
Iterator &operator=(const Iterator &) = default;
|
||
|
|
Iterator &operator=(Iterator &&) = default;
|
||
|
|
Iterator &operator++()
|
||
|
|
{
|
||
|
|
std::advance(scalar_it, Size);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator operator++(int)
|
||
|
|
{
|
||
|
|
Iterator copy(*this);
|
||
|
|
operator++();
|
||
|
|
return copy;
|
||
|
|
}
|
||
|
|
bool operator==(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
#ifndef NDEBUG
|
||
|
|
if (scalar_it == rhs.scalar_it) {
|
||
|
|
return true;
|
||
|
|
} else {
|
||
|
|
T it(scalar_it);
|
||
|
|
for (size_t i = 1; i < Size; ++i) {
|
||
|
|
Vc_ASSERT((++it != rhs.scalar_it));
|
||
|
|
}
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
return scalar_it == rhs.scalar_it;
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
bool operator!=(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
return !operator==(rhs);
|
||
|
|
}
|
||
|
|
pointer operator->() { return scalar_it; }
|
||
|
|
reference operator*() { return scalar_it; }
|
||
|
|
const_pointer operator->() const { return scalar_it; }
|
||
|
|
const_reference operator*() const { return scalar_it; }
|
||
|
|
operator const T &() const { return scalar_it; }
|
||
|
|
protected:
|
||
|
|
T scalar_it;
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
|
||
|
|
class Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
|
||
|
|
: public Iterator<T, N, M, V, Size, std::forward_iterator_tag>
|
||
|
|
{
|
||
|
|
using Base = Iterator<T, N, M, V, Size, std::forward_iterator_tag>;
|
||
|
|
protected:
|
||
|
|
using Base::scalar_it;
|
||
|
|
public:
|
||
|
|
using pointer = typename Base::pointer;
|
||
|
|
using reference = typename Base::reference;
|
||
|
|
using const_pointer = typename Base::const_pointer;
|
||
|
|
using const_reference = typename Base::const_reference;
|
||
|
|
using Iterator<T, N, M, V, Size,
|
||
|
|
std::forward_iterator_tag>::Iterator;
|
||
|
|
Iterator &operator--()
|
||
|
|
{
|
||
|
|
std::advance(scalar_it, -Size);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator operator--(int)
|
||
|
|
{
|
||
|
|
Iterator copy(*this);
|
||
|
|
operator--();
|
||
|
|
return copy;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
|
||
|
|
class Iterator<T, N, M, V, Size, std::random_access_iterator_tag>
|
||
|
|
: public Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
|
||
|
|
{
|
||
|
|
using Base = Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>;
|
||
|
|
protected:
|
||
|
|
using Base::scalar_it;
|
||
|
|
public:
|
||
|
|
using pointer = typename Base::pointer;
|
||
|
|
using reference = typename Base::reference;
|
||
|
|
using const_pointer = typename Base::const_pointer;
|
||
|
|
using const_reference = typename Base::const_reference;
|
||
|
|
using difference_type = typename std::iterator_traits<T>::difference_type;
|
||
|
|
using Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>::
|
||
|
|
Iterator;
|
||
|
|
Iterator &operator+=(difference_type n)
|
||
|
|
{
|
||
|
|
scalar_it += n * difference_type(Size);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator operator+(difference_type n) const { return Iterator(*this) += n; }
|
||
|
|
Iterator &operator-=(difference_type n)
|
||
|
|
{
|
||
|
|
scalar_it -= n * difference_type(Size);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
Iterator operator-(difference_type n) const { return Iterator(*this) -= n; }
|
||
|
|
difference_type operator-(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
constexpr difference_type n = Size;
|
||
|
|
Vc_ASSERT((scalar_it - rhs.scalar_it) % n ==
|
||
|
|
0);
|
||
|
|
return (scalar_it - rhs.scalar_it) / n;
|
||
|
|
}
|
||
|
|
bool operator<(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
return rhs.scalar_it - scalar_it >= difference_type(Size);
|
||
|
|
}
|
||
|
|
bool operator>(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
return scalar_it - rhs.scalar_it >= difference_type(Size);
|
||
|
|
}
|
||
|
|
bool operator<=(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
return rhs.scalar_it - scalar_it >= difference_type(Size) - 1;
|
||
|
|
}
|
||
|
|
bool operator>=(const Iterator &rhs) const
|
||
|
|
{
|
||
|
|
return scalar_it - rhs.scalar_it >= difference_type(Size) - 1;
|
||
|
|
}
|
||
|
|
reference operator[](difference_type i) { return *(*this + i); }
|
||
|
|
const_reference operator[](difference_type i) const { return *(*this + i); }
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
|
||
|
|
Iterator<T, N, M, V, Size, std::random_access_iterator_tag> operator+(
|
||
|
|
typename Iterator<T, N, M, V, Size, std::random_access_iterator_tag>::difference_type
|
||
|
|
n,
|
||
|
|
const Iterator<T, N, M, V, Size, std::random_access_iterator_tag> &i)
|
||
|
|
{
|
||
|
|
return i + n;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
template <typename T, size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<T, N, MT, Category::ForwardIterator>
|
||
|
|
{
|
||
|
|
using type = IteratorDetails::Iterator<T, N>;
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<T, N, MT, Category::BidirectionalIterator>
|
||
|
|
{
|
||
|
|
using type = IteratorDetails::Iterator<T, N>;
|
||
|
|
};
|
||
|
|
template <typename T, size_t N, typename MT>
|
||
|
|
struct ReplaceTypes<T, N, MT, Category::RandomAccessIterator>
|
||
|
|
{
|
||
|
|
using type = IteratorDetails::Iterator<T, N>;
|
||
|
|
};
|
||
|
|
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
|
||
|
|
std::size_t Offset>
|
||
|
|
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
|
||
|
|
conditional_assign(Adapter<S, T, N> &, const M &, const U &)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
|
||
|
|
std::size_t Offset = 0>
|
||
|
|
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
|
||
|
|
conditional_assign(Adapter<S, T, N> &lhs, const M &mask, const U &rhs)
|
||
|
|
{
|
||
|
|
using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
|
||
|
|
using M2 = typename V::mask_type;
|
||
|
|
conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask), get_dispatcher<Offset>(rhs));
|
||
|
|
conditional_assign<Op, S, T, N, M, U, Offset + 1>(lhs, mask, rhs);
|
||
|
|
}
|
||
|
|
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
|
||
|
|
std::size_t Offset>
|
||
|
|
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
|
||
|
|
conditional_assign(Adapter<S, T, N> &, const M &)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
|
||
|
|
std::size_t Offset = 0>
|
||
|
|
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
|
||
|
|
conditional_assign(Adapter<S, T, N> &lhs, const M &mask)
|
||
|
|
{
|
||
|
|
using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
|
||
|
|
using M2 = typename V::mask_type;
|
||
|
|
conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask));
|
||
|
|
conditional_assign<Op, S, T, N, M, Offset + 1>(lhs, mask);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
template <typename T, size_t N = 0, typename MT = void>
|
||
|
|
using simdize = SimdizeDetail::simdize<T, N, MT>;
|
||
|
|
#define Vc_SIMDIZE_INTERFACE(MEMBERS_) \
|
||
|
|
template <std::size_t N_> \
|
||
|
|
inline auto vc_get_()->decltype(std::get<N_>(std::tie MEMBERS_)) \
|
||
|
|
{ \
|
||
|
|
return std::get<N_>(std::tie MEMBERS_); \
|
||
|
|
} \
|
||
|
|
template <std::size_t N_> \
|
||
|
|
inline auto vc_get_() const->decltype(std::get<N_>(std::tie MEMBERS_)) \
|
||
|
|
{ \
|
||
|
|
return std::get<N_>(std::tie MEMBERS_); \
|
||
|
|
} \
|
||
|
|
enum : std::size_t { \
|
||
|
|
tuple_size = std::tuple_size<decltype(std::make_tuple MEMBERS_)>::value \
|
||
|
|
}
|
||
|
|
}
|
||
|
|
namespace std
|
||
|
|
{
|
||
|
|
using Vc::SimdizeDetail::swap;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
namespace Vc_VERSIONED_NAMESPACE
|
||
|
|
{
|
||
|
|
#ifdef DOXYGEN
|
||
|
|
template <class InputIt, class UnaryFunction>
|
||
|
|
UnaryFunction simd_for_each(InputIt first, InputIt last, UnaryFunction f);
|
||
|
|
#else
|
||
|
|
template <class InputIt, class UnaryFunction,
|
||
|
|
class ValueType = typename std::iterator_traits<InputIt>::value_type>
|
||
|
|
inline enable_if<
|
||
|
|
Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
|
||
|
|
UnaryFunction>
|
||
|
|
simd_for_each(InputIt first, InputIt last, UnaryFunction f)
|
||
|
|
{
|
||
|
|
typedef simdize<ValueType> V;
|
||
|
|
typedef simdize<ValueType, 1> V1;
|
||
|
|
const auto lastV = last - V::Size + 1;
|
||
|
|
for (; first < lastV; first += V::Size) {
|
||
|
|
V tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
}
|
||
|
|
for (; first != last; ++first) {
|
||
|
|
V1 tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
}
|
||
|
|
return f;
|
||
|
|
}
|
||
|
|
template <typename InputIt, typename UnaryFunction,
|
||
|
|
class ValueType = typename std::iterator_traits<InputIt>::value_type>
|
||
|
|
inline enable_if<
|
||
|
|
!Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
|
||
|
|
UnaryFunction>
|
||
|
|
simd_for_each(InputIt first, InputIt last, UnaryFunction f)
|
||
|
|
{
|
||
|
|
typedef simdize<ValueType> V;
|
||
|
|
typedef simdize<ValueType, 1> V1;
|
||
|
|
const auto lastV = last - V::size() + 1;
|
||
|
|
for (; first < lastV; first += V::size()) {
|
||
|
|
V tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
store_interleaved(tmp, std::addressof(*first));
|
||
|
|
}
|
||
|
|
for (; first != last; ++first) {
|
||
|
|
V1 tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
store_interleaved(tmp, std::addressof(*first));
|
||
|
|
}
|
||
|
|
return f;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
template <typename InputIt, typename UnaryFunction,
|
||
|
|
class ValueType = typename std::iterator_traits<InputIt>::value_type>
|
||
|
|
inline enable_if<
|
||
|
|
Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
|
||
|
|
UnaryFunction>
|
||
|
|
simd_for_each_n(InputIt first, std::size_t count, UnaryFunction f)
|
||
|
|
{
|
||
|
|
typename std::make_signed<size_t>::type len = count;
|
||
|
|
typedef simdize<ValueType> V;
|
||
|
|
typedef simdize<ValueType, 1> V1;
|
||
|
|
for (; len >= int(V::size()); len -= V::Size, first += V::Size) {
|
||
|
|
V tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
}
|
||
|
|
for (; len != 0; --len, ++first) {
|
||
|
|
V1 tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
}
|
||
|
|
return f;
|
||
|
|
}
|
||
|
|
template <typename InputIt, typename UnaryFunction,
|
||
|
|
class ValueType = typename std::iterator_traits<InputIt>::value_type>
|
||
|
|
inline enable_if<
|
||
|
|
!Traits::is_functor_argument_immutable<UnaryFunction, simdize<ValueType>>::value,
|
||
|
|
UnaryFunction>
|
||
|
|
simd_for_each_n(InputIt first, std::size_t count, UnaryFunction f)
|
||
|
|
{
|
||
|
|
typename std::make_signed<size_t>::type len = count;
|
||
|
|
typedef simdize<ValueType> V;
|
||
|
|
typedef simdize<ValueType, 1> V1;
|
||
|
|
for (; len >= int(V::size()); len -= V::Size, first += V::Size) {
|
||
|
|
V tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
store_interleaved(tmp, std::addressof(*first));
|
||
|
|
}
|
||
|
|
for (; len != 0; --len, ++first) {
|
||
|
|
V1 tmp;
|
||
|
|
load_interleaved(tmp, std::addressof(*first));
|
||
|
|
f(tmp);
|
||
|
|
store_interleaved(tmp, std::addressof(*first));
|
||
|
|
}
|
||
|
|
return f;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#endif
|