library/cpp/sse/ut/test.cpp - main - githubmirror/nbs

library/cpp/sse/ut/test.cpp
2 088 строк69 KB
bulatman
Remove extra semicolon (library)
10 июн 2023, 13:55
10 июн 2023, 13:554683e3d
/*
  Unittests for all SSE instrinsics translated to NEON instrinsics or
  software implementation.
  Should be tested both on Intel and ARM64.
 */
/* Author: Vitaliy Manushkin <agri@yandex-team.ru */

#include <library/cpp/testing/unittest/registar.h>

#include <util/generic/typetraits.h>
#include <util/string/hex.h>
#include <util/random/fast.h>
#include <util/stream/output.h>

#include <algorithm>
#include <array>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>

template <typename TResult, typename TFunc, TFunc* func>
struct T_mm_CallWrapper {
    TResult Value;

    template <typename... TParams>
    T_mm_CallWrapper(TParams&&... params) {
        Value = func(std::forward<TParams>(params)...);
    }

    operator TResult&() {
        return Value;
    }

    operator const TResult&() const {
        return Value;
    }
};

#if defined(_arm64_)
#include "library/cpp/sse/sse2neon.h"
#elif defined(_i386_) || defined(_x86_64_)
#include <xmmintrin.h>
#include <emmintrin.h>
#include <smmintrin.h>
#elif defined(_ppc64_)
#include "library/cpp/sse/powerpc.h"
#else
#error "Unsupported platform"
#endif

#if defined(_arm64_)
#define Wrap(T_mm_func) T_mm_func
#define WrapF(T_mm_func) T_mm_func
#define WrapD(T_mm_func) T_mm_func
#elif defined(_ppc64_) || defined(_i386_) || defined(_x86_64_)
#define Wrap(_mm_func) \
    T_mm_CallWrapper<__m128i, decltype(_mm_func), _mm_func>
#define WrapF(_mm_func) \
    T_mm_CallWrapper<__m128, decltype(_mm_func), _mm_func>
#define WrapD(_mm_func) \
    T_mm_CallWrapper<__m128d, decltype(_mm_func), _mm_func>
using int8x16_t = std::array<i8, 16>;
using int16x8_t = std::array<i16, 8>;
using int32x4_t = std::array<i32, 4>;
using int64x2_t = std::array<i64, 2>;
using uint8x16_t = std::array<ui8, 16>;
using uint16x8_t = std::array<ui16, 8>;
using uint32x4_t = std::array<ui32, 4>;
using uint64x2_t = std::array<ui64, 2>;
using float32x4_t = std::array<float, 4>;
using float64x2_t = std::array<double, 2>;

template <typename TVectorType>
struct TQType {
    static TVectorType As(__m128i param) {
        TVectorType value;
        _mm_storeu_si128((__m128i*)&value, param);
        return value;
    }
    static TVectorType As(__m128 param) {
        TVectorType value;
        _mm_storeu_ps((float*)&value, param);
        return value;
    }
    static TVectorType As(__m128d param) {
        TVectorType value;
        _mm_storeu_pd((double*)&value, param);
        return value;
    }
};
#endif

template <typename TVectorType>
struct TFuncLoad;
template <typename TVectorType>
struct TFuncStore;

template <>
struct TFuncLoad<__m128i> {
    __m128i Value;

    template <typename TPointer>
    TFuncLoad(TPointer* ptr) {
        Value = _mm_loadu_si128((__m128i*)ptr);
    }

    operator __m128i&() {
        return Value;
    }

    operator const __m128i&() const {
        return Value;
    }
};

template <>
struct TFuncLoad<__m128> {
    __m128 Value;

    template <typename TPointer>
    TFuncLoad(TPointer* ptr) {
        Value = _mm_loadu_ps((float*)ptr);
    }

    operator __m128&() {
        return Value;
    }

    operator const __m128&() const {
        return Value;
    }
};

template <>
struct TFuncLoad<__m128d> {
    __m128d Value;

    template <typename TPointer>
    TFuncLoad(TPointer* ptr) {
        Value = _mm_loadu_pd((double*)ptr);
    }

    operator __m128d&() {
        return Value;
    }

    operator const __m128d&() const {
        return Value;
    }
};

template <>
struct TFuncStore<__m128i> {
    template <typename TPointer>
    TFuncStore(TPointer* ptr, __m128i Value) {
        _mm_storeu_si128((__m128i*)ptr, Value);
    }
};

template <>
struct TFuncStore<__m128> {
    template <typename TPointer>
    TFuncStore(TPointer* ptr, __m128 Value) {
        _mm_storeu_ps((float*)ptr, Value);
    }
};

class TSSEEmulTest: public TTestBase {
private:
    UNIT_TEST_SUITE(TSSEEmulTest);
    UNIT_TEST(Test_mm_load_si128);
    UNIT_TEST(Test_mm_loadu_si128);
    UNIT_TEST(Test_mm_storeu_si128);
    UNIT_TEST(Test_mm_loadu_si128_2);
    UNIT_TEST(Test_mm_loadu_ps);
    UNIT_TEST(Test_mm_storeu_ps);

    UNIT_TEST(Test_mm_slli_epi16);
    UNIT_TEST(Test_mm_slli_epi32);
    UNIT_TEST(Test_mm_slli_epi64);
    UNIT_TEST(Test_mm_slli_si128);

    UNIT_TEST(Test_mm_srli_epi16);
    UNIT_TEST(Test_mm_srli_epi32);
    UNIT_TEST(Test_mm_srli_epi64);
    UNIT_TEST(Test_mm_srli_si128);

    UNIT_TEST(Test_mm_srai_epi16);
    UNIT_TEST(Test_mm_srai_epi32);

    UNIT_TEST(Test_mm_sll_epi16);
    UNIT_TEST(Test_mm_sll_epi32);
    UNIT_TEST(Test_mm_sll_epi64);

    UNIT_TEST(Test_mm_srl_epi16);
    UNIT_TEST(Test_mm_srl_epi32);
    UNIT_TEST(Test_mm_srl_epi64);

    UNIT_TEST(Test_mm_add_epi16);
    UNIT_TEST(Test_mm_add_epi32);
    UNIT_TEST(Test_mm_add_epi64);
    UNIT_TEST(Test_mm_add_ps);
    UNIT_TEST(Test_mm_add_pd);

    UNIT_TEST(Test_mm_madd_epi16);

    UNIT_TEST(Test_mm_sub_epi16);
    UNIT_TEST(Test_mm_sub_epi32);
    UNIT_TEST(Test_mm_sub_epi64);
    UNIT_TEST(Test_mm_sub_ps);
    UNIT_TEST(Test_mm_sub_pd);

    UNIT_TEST(Test_mm_mul_ps);
    UNIT_TEST(Test_mm_mul_pd);
    UNIT_TEST(Test_mm_div_ps);
    UNIT_TEST(Test_mm_div_pd);
    UNIT_TEST(Test_mm_max_ps);
    UNIT_TEST(Test_mm_min_ps);
    UNIT_TEST(Test_mm_and_ps);

    UNIT_TEST(Test_mm_unpacklo_epi8);
    UNIT_TEST(Test_mm_unpackhi_epi8);
    UNIT_TEST(Test_mm_unpacklo_epi16);
    UNIT_TEST(Test_mm_unpackhi_epi16);
    UNIT_TEST(Test_mm_unpacklo_epi32);
    UNIT_TEST(Test_mm_unpackhi_epi32);
    UNIT_TEST(Test_mm_unpacklo_epi64);
    UNIT_TEST(Test_mm_unpackhi_epi64);

    UNIT_TEST(Test_mm_or_si128);
    UNIT_TEST(Test_mm_and_si128);
    UNIT_TEST(Test_mm_andnot_si128);

    UNIT_TEST(Test_mm_cmpeq_epi8);
    UNIT_TEST(Test_mm_cmpeq_epi16);
    UNIT_TEST(Test_mm_cmpeq_epi32);
    UNIT_TEST(Test_mm_cmpeq_ps);

    UNIT_TEST(Test_mm_cmpgt_epi8);
    UNIT_TEST(Test_mm_cmpgt_epi16);
    UNIT_TEST(Test_mm_cmpgt_epi32);
    UNIT_TEST(Test_mm_cmpgt_ps);

    UNIT_TEST(Test_mm_cmplt_epi8);
    UNIT_TEST(Test_mm_cmplt_epi16);
    UNIT_TEST(Test_mm_cmplt_epi32);

    UNIT_TEST(Test_mm_set1_epi8);
    UNIT_TEST(Test_mm_set1_epi16);
    UNIT_TEST(Test_mm_set1_epi32);
    UNIT_TEST(Test_mm_set1_ps);
    UNIT_TEST(Test_mm_set_ps1);

    UNIT_TEST(Test_mm_setzero_si128);
    UNIT_TEST(Test_mm_setzero_ps);
    UNIT_TEST(Test_mm_setzero_pd);

    UNIT_TEST(Test_mm_storel_epi64);
    UNIT_TEST(Test_mm_loadl_epi64);

    UNIT_TEST(Test_mm_loadl_pd);
    UNIT_TEST(Test_mm_loadh_pd);
    UNIT_TEST(Test_mm_cvtsd_f64);

    UNIT_TEST(Test_mm_shuffle_epi32);
    UNIT_TEST(Test_mm_movemask_epi8);
    UNIT_TEST(Test_mm_cvtsi128_si32);
    UNIT_TEST(Test_mm_cvtsi128_si64);

    UNIT_TEST(Test_mm_set_epi16);
    UNIT_TEST(Test_mm_set_epi32);
    UNIT_TEST(Test_mm_set_ps);
    UNIT_TEST(Test_mm_set_pd);

    UNIT_TEST(Test_mm_cvtsi32_si128);
    UNIT_TEST(Test_mm_cvtsi64_si128);

    UNIT_TEST(Test_mm_packs_epi16);
    UNIT_TEST(Test_mm_packs_epi32);
    UNIT_TEST(Test_mm_packus_epi16);

    UNIT_TEST(Test_mm_extract_epi16);
    UNIT_TEST(Test_mm_extract_epi8);
    UNIT_TEST(Test_mm_extract_epi32);
    UNIT_TEST(Test_mm_extract_epi64);

    UNIT_TEST(Test_MM_TRANSPOSE4_PS);
    UNIT_TEST(Test_mm_movemask_ps);
    UNIT_TEST(Test_mm_movemask_ps_2);

    UNIT_TEST(Test_mm_cvtepi32_ps);
    UNIT_TEST(Test_mm_cvtps_epi32);
    UNIT_TEST(Test_mm_cvttps_epi32);

    UNIT_TEST(Test_mm_castsi128_ps);
    UNIT_TEST(Test_mm_castps_si128);

    UNIT_TEST(Test_mm_mul_epu32);

    UNIT_TEST(Test_mm_cmpunord_ps);
    UNIT_TEST(Test_mm_andnot_ps);
    UNIT_TEST(Test_mm_shuffle_ps);
    UNIT_TEST(Test_mm_shuffle_pd);
    UNIT_TEST(Test_mm_or_ps);
    UNIT_TEST(Test_mm_store_ss);
    UNIT_TEST(Test_mm_store_ps);
    UNIT_TEST(Test_mm_storeu_pd);
    UNIT_TEST(Test_mm_loadu_pd);
    UNIT_TEST(Test_mm_rsqrt_ps);
    UNIT_TEST(Test_matrixnet_powerpc);

    UNIT_TEST_SUITE_END();

public:
    void Test_mm_load_si128();
    void Test_mm_loadu_si128();
    void Test_mm_storeu_si128();
    void Test_mm_loadu_si128_2();
    void Test_mm_loadu_ps();
    void Test_mm_storeu_ps();

    template <typename TElem, int bits, int elemCount,
              typename TFunc, typename TShifter, typename TOp, typename TElemFunc>
    void Test_mm_shifter_epiXX();

    enum class EDirection {
        Left,
        Right
    };

    struct TShiftRes {
        __m128i Value[17];
    };

    void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo);

    void Test_mm_slli_epi16();
    void Test_mm_slli_epi32();
    void Test_mm_slli_epi64();
    void Test_mm_slli_si128();

    void Test_mm_srli_epi16();
    void Test_mm_srli_epi32();
    void Test_mm_srli_epi64();
    void Test_mm_srli_si128();

    void Test_mm_srai_epi16();
    void Test_mm_srai_epi32();

    void Test_mm_sll_epi16();
    void Test_mm_sll_epi32();
    void Test_mm_sll_epi64();

    void Test_mm_srl_epi16();
    void Test_mm_srl_epi32();
    void Test_mm_srl_epi64();

    void Test_mm_add_epi8();
    void Test_mm_add_epi16();
    void Test_mm_add_epi32();
    void Test_mm_add_epi64();
    void Test_mm_add_ps();
    void Test_mm_add_pd();

    void Test_mm_madd_epi16();

    void Test_mm_sub_epi8();
    void Test_mm_sub_epi16();
    void Test_mm_sub_epi32();
    void Test_mm_sub_epi64();
    void Test_mm_sub_ps();
    void Test_mm_sub_pd();

    void Test_mm_mul_ps();
    void Test_mm_mul_pd();
    void Test_mm_div_ps();
    void Test_mm_div_pd();
    void Test_mm_max_ps();
    void Test_mm_min_ps();
    void Test_mm_and_ps();

    template <typename TElem, int bits, int elemCount, int shift,
              typename TFunc, typename TOp>
    void Test_mm_unpack_epiXX();
    void Test_mm_unpacklo_epi8();
    void Test_mm_unpackhi_epi8();
    void Test_mm_unpacklo_epi16();
    void Test_mm_unpackhi_epi16();
    void Test_mm_unpacklo_epi32();
    void Test_mm_unpackhi_epi32();
    void Test_mm_unpacklo_epi64();
    void Test_mm_unpackhi_epi64();

    template <typename TElem, unsigned elemCount,
              typename TFunc, typename TElemFunc,
              typename TOp, typename TVectorType = __m128i>
    void Test_mm_dualop();

    template <typename TElem, unsigned elemCount,
              typename TFunc, typename TElemFunc,
              typename TOp, typename TVectorType = __m128i>
    void Test_mm_dualcmp();

    void Test_mm_or_si128();
    void Test_mm_and_si128();
    void Test_mm_andnot_si128();

    void Test_mm_cmpeq_epi8();
    void Test_mm_cmpeq_epi16();
    void Test_mm_cmpeq_epi32();
    void Test_mm_cmpeq_ps();

    void Test_mm_cmpgt_epi8();
    void Test_mm_cmpgt_epi16();
    void Test_mm_cmpgt_epi32();
    void Test_mm_cmpgt_ps();

    void Test_mm_cmplt_epi8();
    void Test_mm_cmplt_epi16();
    void Test_mm_cmplt_epi32();

    template <typename TElem, int elemCount,
              typename TFunc, typename TOp, typename TVectorType>
    void Test_mm_setter_epiXX();
    void Test_mm_set1_epi8();
    void Test_mm_set1_epi16();
    void Test_mm_set1_epi32();
    void Test_mm_set1_ps();
    void Test_mm_set_ps1();

    void Test_mm_setzero_si128();
    void Test_mm_setzero_ps();
    void Test_mm_setzero_pd();

    void Test_mm_loadl_epi64();
    void Test_mm_storel_epi64();

    void Test_mm_loadl_pd();
    void Test_mm_loadh_pd();
    void Test_mm_cvtsd_f64();

    void Test_mm_shuffle_epi32();
    void Test_mm_movemask_epi8();
    void Test_mm_cvtsi128_si32();
    void Test_mm_cvtsi128_si64();

    void Test_mm_set_epi16();
    void Test_mm_set_epi32();
    void Test_mm_set_ps();
    void Test_mm_set_pd();

    void Test_mm_cvtsi32_si128();
    void Test_mm_cvtsi64_si128();

    template <typename TElem, typename TNarrow, unsigned elemCount,
              typename TFunc>
    void Test_mm_packs_epiXX();
    void Test_mm_packs_epi16();
    void Test_mm_packs_epi32();
    void Test_mm_packus_epi16();

    void Test_mm_extract_epi16();
    void Test_mm_extract_epi8();
    void Test_mm_extract_epi32();
    void Test_mm_extract_epi64();

    void Test_MM_TRANSPOSE4_PS();
    void Test_mm_movemask_ps();
    void Test_mm_movemask_ps_2();

    template <typename TFrom, typename TTo, unsigned elemCount,
              typename TLoadVector, typename TResultVector,
              typename TElemFunc, typename TFunc, typename TOp>
    void Test_mm_convertop();
    void Test_mm_cvtepi32_ps();
    void Test_mm_cvtps_epi32();
    void Test_mm_cvttps_epi32();

    template <typename TLoadVector, typename TCastVector,
              typename TFunc, TFunc* func>
    void Test_mm_castXX();
    void Test_mm_castsi128_ps();
    void Test_mm_castps_si128();

    void Test_mm_mul_epu32();

    void Test_mm_cmpunord_ps();
    void Test_mm_store_ss();
    void Test_mm_store_ps();
    void Test_mm_storeu_pd();
    void Test_mm_andnot_ps();
    void Test_mm_shuffle_ps();
    void Test_mm_shuffle_pd();
    void Test_mm_or_ps();
    void Test_mm_loadu_pd();
    void Test_mm_rsqrt_ps();
    void Test_mm_rsqrt_ss();
    void Test_matrixnet_powerpc();
};

UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest);

void TSSEEmulTest::Test_mm_load_si128() {
    alignas(16) char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    __m128i value = _mm_load_si128((__m128i*)&data);
    UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[0], 0xAABB2211CCFF00AAUL);
    UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[1], 0x1C66775588449933UL);
}

void TSSEEmulTest::Test_mm_loadu_si128() {
    alignas(16) char data[17] = {
        '\x66',
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    UNIT_ASSERT((ui64(&data[1]) & 0x1) == 0x1);
    __m128i value = _mm_loadu_si128((__m128i*)&data[1]);
    UNIT_ASSERT(TQType<uint64x2_t>::As(value)[0] == 0xAABB2211CCFF00AAUL);
    UNIT_ASSERT(TQType<uint64x2_t>::As(value)[1] == 0x1C66775588449933UL);
}

void TSSEEmulTest::Test_mm_storeu_si128() {
    alignas(16) unsigned char stub[32] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
    };

    alignas(16) unsigned char value[16] = {
        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf
    };

    const __m128i val = _mm_loadu_si128((__m128i*)&value[0]);

    for (size_t shift = 0; shift != 17; ++shift) {
        alignas(16) unsigned char res[sizeof(stub)];
        memcpy(res, stub, sizeof(res));

        _mm_storeu_si128((__m128i*)&res[shift], val);


        alignas(16) unsigned char etalon[sizeof(stub)];
        memcpy(etalon, stub, sizeof(etalon));
        for (size_t i = 0; i != sizeof(value); ++i) {
            etalon[shift + i] = value[i];
        }

        for (size_t i = 0; i != sizeof(etalon) / sizeof(etalon[0]); ++i) {
            UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, 32) << " vs etalon: " << HexEncode(etalon, 32));
        }
    }

}


void TSSEEmulTest::Test_mm_loadu_si128_2() {
    alignas(16) unsigned char stub[32] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
    };

    for (size_t shift = 0; shift != 17; ++shift) {
        const __m128i val = _mm_loadu_si128((const __m128i*)&stub[shift]);
        alignas(16) unsigned char res[16];
        _mm_store_si128((__m128i*)res, val);

        for (size_t i = 0; i != 16; ++i) {
            UNIT_ASSERT_EQUAL_C(res[i], stub[i + shift], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
        }
    }
}


void TSSEEmulTest::Test_mm_loadu_ps() {
    alignas(16) float stub[8] = {
        0.f, 1.f, 2.f, 3.f,
        4.f, 5.f, 6.f, 7.f
    };

    for (size_t shift = 0; shift != 5; ++shift) {
        const __m128 val = _mm_loadu_ps(&stub[shift]);
        alignas(16) float res[4];
        _mm_store_ps(res, val);

        for (size_t i = 0; i != 4; ++i) {
            UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
        }
    }
}


void TSSEEmulTest::Test_mm_storeu_ps() {
    alignas(16) float stub[8] = {
        0.f, 1.f, 2.f, 3.f,
        4.f, 5.f, 6.f, 7.f
    };

    alignas(16) float value[4] = {
        100.f, 101.f, 102.f, 103.f
    };
    const __m128 val = _mm_load_ps(value);

    for (size_t shift = 0; shift != 5; ++shift) {
        alignas(16) float res[sizeof(stub) / sizeof(stub[0])];
        memcpy(res, stub, sizeof(stub));

        _mm_storeu_ps(&res[shift], val);

        float etalon[sizeof(stub) / sizeof(stub[0])];
        memcpy(etalon, stub, sizeof(stub));
        for (size_t i = 0; i != 4; ++i) {
            etalon[i + shift] = value[i];
        }

        for (size_t i = 0; i != sizeof(stub) / sizeof(stub[0]); ++i) {
            UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, sizeof(res)) << " vs etalon: " << HexEncode(etalon, sizeof(etalon)));
        }
    }
}

template<typename C>
C MakeNumber(unsigned number);

template<>
__m128i MakeNumber<__m128i>(unsigned number) {
    char data[16] = {0};
    memcpy(data, &number, sizeof(number));

    return _mm_loadu_si128((__m128i*)data);
}

template<>
unsigned MakeNumber<unsigned>(unsigned number) {
    return number;
}

template <typename TElem, int bits, int elemCount,
          typename TFunc, typename TShifter, typename TOp, typename TElemFunc>
void TSSEEmulTest::Test_mm_shifter_epiXX() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    TElem* dataw = reinterpret_cast<TElem*>(&data);

    __m128i value = _mm_loadu_si128((__m128i*)&data);

    for (unsigned shifter = 0; shifter <= bits; ++shifter) {
        TElem shiftedData[elemCount];
        for (unsigned i = 0; i < elemCount; ++i) {
            shiftedData[i] = TElemFunc::Call(dataw[i], shifter);
        }

        const TShifter adhoc_shifter = MakeNumber<TShifter>(shifter);

        __m128i result = TFunc(value, adhoc_shifter);

        for (unsigned i = 0; i < elemCount; ++i) {
            UNIT_ASSERT_EQUAL(shiftedData[i], TQType<TOp>::As(result)[i]);
        }
    }
}


void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) {
    const char data[48] = {
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C',
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'
    };

    const __m128i a = _mm_loadu_si128((__m128i*)(data + 16));
    const TShiftRes res = foo(a);
    for (int shift = 0; shift <= 16; ++shift) {
        const int etalon_offset = 16 + (direction == EDirection::Left ? -shift : shift); //!< specific to little endian byte order.
        const char* etalon = data + etalon_offset;
        const char* res_bytes = (const char*)&res.Value[shift];

        for (size_t byte = 0; byte != 16; ++byte) {
            UNIT_ASSERT_EQUAL(etalon[byte], res_bytes[byte]);
        }
    }
}

template <typename TElem>
struct THelperASHR {
    static TElem Call(const TElem op, const int shift) {
        constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
        if (op < 0) {
            // Arithmetic shift propagates sign bit to the right
            // while operator>> is implementation defined for negative values,
            // so we can't use it as a reference implementation
            // and we need to write some standard consistent code.
            typedef TFixedWidthUnsignedInt<TElem> TUnsignedElem;
            TUnsignedElem uOp(op);
            const TUnsignedElem signBit = TUnsignedElem(1) << (nBitsInOp - 1);
            Y_ENSURE(shift >= 0);
            for (int i = 0; i != shift; ++i) {
                uOp = signBit | (uOp >> 1);
            }
            // unsigned -> signed conversion is also implementation defined, so we need to use some other method.
            return reinterpret_cast<TElem&>(uOp);
        }
        return shift < nBitsInOp ? op >> shift : 0;
    }
};

template <typename TElem>
struct THelperSHR {
    static TElem Call(const TElem op, const int shift) {
        constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
        return shift < nBitsInOp ? op >> shift : 0;
    }
};

void TSSEEmulTest::Test_mm_srli_epi16() {
    Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_srli_epi16), unsigned, uint16x8_t,
                          THelperSHR<ui16>>();
}

void TSSEEmulTest::Test_mm_srli_epi32() {
    Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_srli_epi32), unsigned, uint32x4_t,
                          THelperSHR<ui32>>();
}

void TSSEEmulTest::Test_mm_srli_epi64() {
    Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_srli_epi64), unsigned, uint64x2_t,
                          THelperSHR<ui64>>();
}

template <typename TElem>
struct THelperSHL {
    static TElem Call(const TElem op, const int shift) {
        constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
        return shift < nBitsInOp ? op << shift : 0;
    }
};

void TSSEEmulTest::Test_mm_slli_epi16() {
    Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_slli_epi16), unsigned, uint16x8_t,
                          THelperSHL<ui16>>();
}

void TSSEEmulTest::Test_mm_slli_epi32() {
    Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_slli_epi32), unsigned, uint32x4_t,
                          THelperSHL<ui32>>();
}

void TSSEEmulTest::Test_mm_slli_epi64() {
    Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_slli_epi64), unsigned, uint64x2_t,
                          THelperSHL<ui64>>();
}

void TSSEEmulTest::Test_mm_slli_si128() {
    Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes {
        TShiftRes res;
        res.Value[0] = _mm_slli_si128(a, 0);
        res.Value[1] = _mm_slli_si128(a, 1);
        res.Value[2] = _mm_slli_si128(a, 2);
        res.Value[3] = _mm_slli_si128(a, 3);
        res.Value[4] = _mm_slli_si128(a, 4);
        res.Value[5] = _mm_slli_si128(a, 5);
        res.Value[6] = _mm_slli_si128(a, 6);
        res.Value[7] = _mm_slli_si128(a, 7);
        res.Value[8] = _mm_slli_si128(a, 8);
        res.Value[9] = _mm_slli_si128(a, 9);
        res.Value[10] = _mm_slli_si128(a, 10);
        res.Value[11] = _mm_slli_si128(a, 11);
        res.Value[12] = _mm_slli_si128(a, 12);
        res.Value[13] = _mm_slli_si128(a, 13);
        res.Value[14] = _mm_slli_si128(a, 14);
        res.Value[15] = _mm_slli_si128(a, 15);
        res.Value[16] = _mm_slli_si128(a, 16);

        return res;
    });
}

void TSSEEmulTest::Test_mm_srl_epi16() {
    Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi16), _mm_srl_epi16>, __m128i, uint16x8_t,
                          THelperSHR<ui16>>();
}

void TSSEEmulTest::Test_mm_srl_epi32() {
    Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi32), _mm_srl_epi32>, __m128i, uint32x4_t,
                          THelperSHR<ui32>>();
}

void TSSEEmulTest::Test_mm_srl_epi64() {
    Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi64), _mm_srl_epi64>, __m128i, uint64x2_t,
                          THelperSHR<ui64>>();
}

void TSSEEmulTest::Test_mm_srai_epi16() {
    Test_mm_shifter_epiXX<i16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi16), _mm_srai_epi16>, unsigned, int16x8_t,
                          THelperASHR<i16>>();
}

void TSSEEmulTest::Test_mm_srai_epi32() {
    Test_mm_shifter_epiXX<i32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi32), _mm_srai_epi32>, unsigned, int32x4_t,
                          THelperASHR<i32>>();
}

void TSSEEmulTest::Test_mm_srli_si128() {
    Test_mm_byte_shifter(EDirection::Right, [](__m128i a) -> TShiftRes {
        TShiftRes res;
        res.Value[0] = _mm_srli_si128(a, 0);
        res.Value[1] = _mm_srli_si128(a, 1);
        res.Value[2] = _mm_srli_si128(a, 2);
        res.Value[3] = _mm_srli_si128(a, 3);
        res.Value[4] = _mm_srli_si128(a, 4);
        res.Value[5] = _mm_srli_si128(a, 5);
        res.Value[6] = _mm_srli_si128(a, 6);
        res.Value[7] = _mm_srli_si128(a, 7);
        res.Value[8] = _mm_srli_si128(a, 8);
        res.Value[9] = _mm_srli_si128(a, 9);
        res.Value[10] = _mm_srli_si128(a, 10);
        res.Value[11] = _mm_srli_si128(a, 11);
        res.Value[12] = _mm_srli_si128(a, 12);
        res.Value[13] = _mm_srli_si128(a, 13);
        res.Value[14] = _mm_srli_si128(a, 14);
        res.Value[15] = _mm_srli_si128(a, 15);
        res.Value[16] = _mm_srli_si128(a, 16);

        return res;
    });
}

void TSSEEmulTest::Test_mm_sll_epi16() {
    Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi16), _mm_sll_epi16>, __m128i, uint16x8_t,
                          THelperSHL<ui16>>();
}

void TSSEEmulTest::Test_mm_sll_epi32() {
    Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi32), _mm_sll_epi32>, __m128i, uint32x4_t,
                          THelperSHL<ui32>>();
}

void TSSEEmulTest::Test_mm_sll_epi64() {
    Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi64), _mm_sll_epi64>, __m128i, uint64x2_t,
                          THelperSHL<ui64>>();
}

template <typename TElem>
struct THelperAdd {
    static TElem Call(const TElem op1, const TElem op2) {
        return op1 + op2;
    }
};

void TSSEEmulTest::Test_mm_add_epi16() {
    Test_mm_dualop<ui16, 8, Wrap(_mm_add_epi16), THelperAdd<ui16>, uint16x8_t>();
}

void TSSEEmulTest::Test_mm_add_epi32() {
    Test_mm_dualop<ui32, 4, Wrap(_mm_add_epi32), THelperAdd<ui32>, uint32x4_t>();
}

void TSSEEmulTest::Test_mm_add_epi64() {
    Test_mm_dualop<ui64, 2, Wrap(_mm_add_epi64), THelperAdd<ui64>, uint64x2_t>();
}

void TSSEEmulTest::Test_mm_add_ps() {
    Test_mm_dualop<float, 2, WrapF(_mm_add_ps),
                   THelperAdd<float>, float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_add_pd() {
    Test_mm_dualop<double, 2, WrapD(_mm_add_pd),
                   THelperAdd<double>, float64x2_t, __m128d>();
}

void TSSEEmulTest::Test_mm_madd_epi16() {
    alignas(16) const char data1[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'
    };
    alignas(16) const char data2[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'
    };

    const __m128i value1 = TFuncLoad<__m128i>(&data1);
    const __m128i value2 = TFuncLoad<__m128i>(&data2);
    const __m128i res = _mm_madd_epi16(value1, value2);

    const i16* dataw1 = reinterpret_cast<const i16*>(&data1);
    const i16* dataw2 = reinterpret_cast<const i16*>(&data2);

    for (size_t i = 0; i != 4; ++i) {
        const size_t dataIdx = i * 2;
        const i32 etalonResult = (i32) dataw1[dataIdx] * (i32) dataw2[dataIdx] + (i32) dataw1[dataIdx + 1] * (i32) dataw2[dataIdx + 1];
        const i32 value = TQType<int32x4_t>::As(res)[i];
        UNIT_ASSERT_EQUAL(value, etalonResult);
    }
}


template <typename TElem>
struct THelperSub {
    static TElem Call(const TElem op1, const TElem op2) {
        return op1 - op2;
    }
};

void TSSEEmulTest::Test_mm_sub_epi16() {
    Test_mm_dualop<ui16, 8, Wrap(_mm_sub_epi16), THelperSub<ui16>, uint16x8_t>();
}

void TSSEEmulTest::Test_mm_sub_epi32() {
    Test_mm_dualop<ui32, 4, Wrap(_mm_sub_epi32), THelperSub<ui32>, uint32x4_t>();
}

void TSSEEmulTest::Test_mm_sub_epi64() {
    Test_mm_dualop<ui64, 2, Wrap(_mm_sub_epi64), THelperSub<ui64>, uint64x2_t>();
}

void TSSEEmulTest::Test_mm_sub_ps() {
    Test_mm_dualop<float, 4, WrapF(_mm_sub_ps), THelperSub<float>,
                   float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_sub_pd() {
    Test_mm_dualop<double, 2, WrapD(_mm_sub_pd), THelperSub<double>,
                   float64x2_t, __m128d>();
}

void TSSEEmulTest::Test_mm_mul_ps() {
    struct THelper {
        static float Call(const float op1, const float op2) {
            return op1 * op2;
        }
    };
    Test_mm_dualop<float, 4, WrapF(_mm_mul_ps), THelper, float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_mul_pd() {
    struct THelper {
        static double Call(const double op1, const double op2) {
            return op1 * op2;
        }
    };
    Test_mm_dualop<double, 2, WrapD(_mm_mul_pd), THelper, float64x2_t, __m128d>();
}

void TSSEEmulTest::Test_mm_div_ps() {
    struct THelper {
        static float Call(const float op1, const float op2) {
            return op1 / op2;
        }
    };
    Test_mm_dualop<float, 4, WrapF(_mm_div_ps), THelper, float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_div_pd() {
    struct THelper {
        static double Call(const double op1, const double op2) {
            return op1 / op2;
        }
    };
    Test_mm_dualop<double, 2, WrapD(_mm_div_pd), THelper, float64x2_t, __m128d>();
}

void TSSEEmulTest::Test_mm_max_ps() {
    struct THelper {
        static float Call(const float op1, const float op2) {
            return std::max(op1, op2);
        }
    };
    Test_mm_dualop<float, 4, WrapF(_mm_max_ps), THelper, float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_min_ps() {
    struct THelper {
        static float Call(const float op1, const float op2) {
            return std::min(op1, op2);
        }
    };
    Test_mm_dualop<float, 4, WrapF(_mm_min_ps), THelper, float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_and_ps() {
    struct THelper {
        static float Call(const float op1, const float op2) {
            union Cast {
                unsigned int AsUInt;
                float AsFloat;
            };
            Cast v1, v2, result;
            v1.AsFloat = op1;
            v2.AsFloat = op2;
            result.AsUInt = v1.AsUInt & v2.AsUInt;
            return result.AsFloat;
        }
    };
    Test_mm_dualcmp<float, 4, WrapF(_mm_and_ps),
                    THelper, float32x4_t, __m128>();
}

template <typename TElem, int bits, int elemCount, int shift,
          typename TFunc, typename TOp>
void TSSEEmulTest::Test_mm_unpack_epiXX() {
    char data1[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    char data2[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
    TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
    TElem* dataw2 = reinterpret_cast<TElem*>(&data2);

    __m128i value1 = _mm_loadu_si128((__m128i*)&data1);
    __m128i value2 = _mm_loadu_si128((__m128i*)&data2);

    TElem zippedData[elemCount];
    for (unsigned i = 0; i < elemCount / 2; ++i) {
        zippedData[i * 2] = dataw1[i + shift];
        zippedData[i * 2 + 1] = dataw2[i + shift];
    }
    __m128i result = TFunc(value1, value2);

    for (unsigned i = 0; i < elemCount / 2; ++i) {
        UNIT_ASSERT_EQUAL(zippedData[i * 2], TQType<TOp>::As(result)[i * 2]);
        UNIT_ASSERT_EQUAL(zippedData[i * 2 + 1],
                          TQType<TOp>::As(result)[i * 2 + 1]);
    }
}

void TSSEEmulTest::Test_mm_unpacklo_epi8() {
    Test_mm_unpack_epiXX<ui8, 8, 16, 0, Wrap(_mm_unpacklo_epi8), uint8x16_t>();
}

void TSSEEmulTest::Test_mm_unpackhi_epi8() {
    Test_mm_unpack_epiXX<ui8, 8, 16, 8, Wrap(_mm_unpackhi_epi8), uint8x16_t>();
}

void TSSEEmulTest::Test_mm_unpacklo_epi16() {
    Test_mm_unpack_epiXX<ui16, 16, 8, 0, Wrap(_mm_unpacklo_epi16), uint16x8_t>();
}

void TSSEEmulTest::Test_mm_unpackhi_epi16() {
    Test_mm_unpack_epiXX<ui16, 16, 8, 4, Wrap(_mm_unpackhi_epi16), uint16x8_t>();
}

void TSSEEmulTest::Test_mm_unpacklo_epi32() {
    Test_mm_unpack_epiXX<ui32, 32, 4, 0, Wrap(_mm_unpacklo_epi32), uint32x4_t>();
}

void TSSEEmulTest::Test_mm_unpackhi_epi32() {
    Test_mm_unpack_epiXX<ui32, 32, 4, 2, Wrap(_mm_unpackhi_epi32), uint32x4_t>();
}

void TSSEEmulTest::Test_mm_unpacklo_epi64() {
    Test_mm_unpack_epiXX<ui64, 64, 2, 0, Wrap(_mm_unpacklo_epi64), uint64x2_t>();
}

void TSSEEmulTest::Test_mm_unpackhi_epi64() {
    Test_mm_unpack_epiXX<ui64, 64, 2, 1, Wrap(_mm_unpackhi_epi64), uint64x2_t>();
}

template <typename TElem, unsigned elemCount,
          typename TFunc, typename TElemFunc,
          typename TOp, typename TVectorType>
void TSSEEmulTest::Test_mm_dualop() {
    char data1[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    char data2[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
    TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
    TElem* dataw2 = reinterpret_cast<TElem*>(&data2);

    TVectorType value1 = TFuncLoad<TVectorType>(&data1);
    TVectorType value2 = TFuncLoad<TVectorType>(&data2);

    TElem procData[elemCount];
    for (unsigned i = 0; i < elemCount; ++i) {
        procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]);
    }
    TVectorType result = TFunc(value1, value2);

    for (unsigned i = 0; i < elemCount; ++i) {
        UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]);
    }
}

/* This is almost the same as Test_mm_dualop,
   but different data1 and data2 */
template <typename TElem, unsigned elemCount,
          typename TFunc, typename TElemFunc,
          typename TOp, typename TVectorType>
void TSSEEmulTest::Test_mm_dualcmp() {
    char data1[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x66', '\x77', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C'};
    char data2[16] = {
        '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
    TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
    TElem* dataw2 = reinterpret_cast<TElem*>(&data2);

    TVectorType value1 = TFuncLoad<TVectorType>(&data1);
    TVectorType value2 = TFuncLoad<TVectorType>(&data2);

    TElem procData[elemCount];
    for (unsigned i = 0; i < elemCount; ++i) {
        procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]);
    }
    TVectorType result = TFunc(value1, value2);

    for (unsigned i = 0; i < elemCount; ++i) {
        /* memcmp is for compare to invalid floats in results */
        const TElem value = TQType<TOp>::As(result)[i];
        UNIT_ASSERT(memcmp(&(procData[i]), &value, sizeof(TElem)) == 0);
    }
}

void TSSEEmulTest::Test_mm_or_si128() {
    struct THelper {
        static ui64 Call(const ui64 op1, const ui64 op2) {
            return op1 | op2;
        }
    };

    Test_mm_dualop<ui64, 2, Wrap(_mm_or_si128), THelper, uint64x2_t>();
}

void TSSEEmulTest::Test_mm_and_si128() {
    struct THelper {
        static ui64 Call(const ui64 op1, const ui64 op2) {
            return op1 & op2;
        }
    };

    Test_mm_dualop<ui64, 2, Wrap(_mm_and_si128), THelper, uint64x2_t>();
}

void TSSEEmulTest::Test_mm_andnot_si128() {
    struct THelper {
        static ui64 Call(const ui64 op1, const ui64 op2) {
            return (~op1) & op2;
        }
    };

    Test_mm_dualop<ui64, 2, Wrap(_mm_andnot_si128), THelper, uint64x2_t>();
}

template <typename TElem>
struct THelperCMPEQ {
    static TElem Call(const TElem op1, const TElem op2) {
        return op1 == op2 ? ~TElem(0) : TElem(0);
    }
};

void TSSEEmulTest::Test_mm_cmpeq_epi8() {
    Test_mm_dualcmp<ui8, 16, Wrap(_mm_cmpeq_epi8),
                    THelperCMPEQ<ui8>, uint8x16_t>();
}

void TSSEEmulTest::Test_mm_cmpeq_epi16() {
    Test_mm_dualcmp<ui16, 8, Wrap(_mm_cmpeq_epi16),
                    THelperCMPEQ<ui16>, uint16x8_t>();
}

void TSSEEmulTest::Test_mm_cmpeq_epi32() {
    Test_mm_dualcmp<ui32, 4, Wrap(_mm_cmpeq_epi32),
                    THelperCMPEQ<ui32>, uint32x4_t>();
}

void TSSEEmulTest::Test_mm_cmpeq_ps() {
    struct THelperFloat {
        static float Call(const float op1, const float op2) {
            union Cast {
                unsigned int AsUInt;
                float AsFloat;
            };
            Cast value;
            value.AsUInt = op1 == op2 ? 0xFFFFFFFF : 0;
            return value.AsFloat;
        }
    };

    Test_mm_dualcmp<float, 4, WrapF(_mm_cmpeq_ps),
                    THelperFloat, float32x4_t, __m128>();
}

template <typename TElem>
struct THelperCMPGT {
    static TElem Call(const TElem op1, const TElem op2) {
        return op1 > op2 ? ~TElem(0) : TElem(0);
    }
};

void TSSEEmulTest::Test_mm_cmpgt_epi8() {
    Test_mm_dualcmp<i8, 16, Wrap(_mm_cmpgt_epi8),
                    THelperCMPGT<i8>, int8x16_t>();
}

void TSSEEmulTest::Test_mm_cmpgt_epi16() {
    Test_mm_dualcmp<i16, 8, Wrap(_mm_cmpgt_epi16),
                    THelperCMPGT<i16>, int16x8_t>();
}

void TSSEEmulTest::Test_mm_cmpgt_epi32() {
    Test_mm_dualcmp<i32, 4, Wrap(_mm_cmpgt_epi32),
                    THelperCMPGT<i32>, int32x4_t>();
}

void TSSEEmulTest::Test_mm_cmpgt_ps() {
    struct THelperFloat {
        static float Call(const float op1, const float op2) {
            union Cast {
                unsigned int AsUInt;
                float AsFloat;
            };
            Cast value;
            value.AsUInt = op1 > op2 ? 0xFFFFFFFF : 0;
            return value.AsFloat;
        }
    };

    Test_mm_dualcmp<float, 4, WrapF(_mm_cmpgt_ps),
                    THelperFloat, float32x4_t, __m128>();
}

template <typename TElem>
struct THelperCMPLT {
    static TElem Call(const TElem op1, const TElem op2) {
        return op1 < op2 ? ~TElem(0) : TElem(0);
    }
};

void TSSEEmulTest::Test_mm_cmplt_epi8() {
    Test_mm_dualcmp<i8, 16, Wrap(_mm_cmplt_epi8),
                    THelperCMPLT<i8>, int8x16_t>();
}

void TSSEEmulTest::Test_mm_cmplt_epi16() {
    Test_mm_dualcmp<i16, 8, Wrap(_mm_cmplt_epi16),
                    THelperCMPLT<i16>, int16x8_t>();
}

void TSSEEmulTest::Test_mm_cmplt_epi32() {
    Test_mm_dualcmp<i32, 4, Wrap(_mm_cmplt_epi32),
                    THelperCMPLT<i32>, int32x4_t>();
}

template <typename TElem, int elemCount,
          typename TFunc, typename TOp, typename TVectorType>
void TSSEEmulTest::Test_mm_setter_epiXX() {
    char data[64] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
        '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
    TElem* dataw = reinterpret_cast<TElem*>(&data);

    for (unsigned dataItem = 0; dataItem < elemCount * 4; ++dataItem) {
        TVectorType value = TFunc(dataw[dataItem]);

        for (unsigned i = 0; i < elemCount; ++i)
            UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<TOp>::As(value)[i]);
    }
}

void TSSEEmulTest::Test_mm_set1_epi8() {
    Test_mm_setter_epiXX<i8, 16, Wrap(_mm_set1_epi8), int8x16_t, __m128i>();
}
void TSSEEmulTest::Test_mm_set1_epi16() {
    Test_mm_setter_epiXX<i16, 8, Wrap(_mm_set1_epi16), int16x8_t, __m128i>();
}
void TSSEEmulTest::Test_mm_set1_epi32() {
    Test_mm_setter_epiXX<i32, 4, Wrap(_mm_set1_epi32), int32x4_t, __m128i>();
}
void TSSEEmulTest::Test_mm_set1_ps() {
    Test_mm_setter_epiXX<float, 4, WrapF(_mm_set1_ps), float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_set_ps1() {
    Test_mm_setter_epiXX<float, 4, WrapF(_mm_set_ps1), float32x4_t, __m128>();
}

void TSSEEmulTest::Test_mm_setzero_si128() {
    __m128i value = _mm_setzero_si128();
    for (unsigned i = 0; i < 4; ++i)
        UNIT_ASSERT_EQUAL(0, TQType<uint32x4_t>::As(value)[i]);
}

void TSSEEmulTest::Test_mm_setzero_ps() {
    __m128 value = _mm_setzero_ps();
    for (unsigned i = 0; i < 4; ++i)
        UNIT_ASSERT_EQUAL(0.0, TQType<float32x4_t>::As(value)[i]);
}

void TSSEEmulTest::Test_mm_setzero_pd() {
    __m128d value = _mm_setzero_pd();
    for (unsigned i = 0; i < 2; ++i)
        UNIT_ASSERT_EQUAL(0.0, TQType<float64x2_t>::As(value)[i]);
}

void TSSEEmulTest::Test_mm_loadl_epi64() {
    char data[64] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
        '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
    ui64* dataw = reinterpret_cast<ui64*>(&data);

    for (unsigned dataItem = 0; dataItem < 8; ++dataItem) {
        __m128i value = _mm_loadl_epi64((__m128i const*)&dataw[dataItem]);

        UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<uint64x2_t>::As(value)[0]);
        UNIT_ASSERT_EQUAL(0, TQType<uint64x2_t>::As(value)[1]);
    }
}

void TSSEEmulTest::Test_mm_storel_epi64() {
    char data[64] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
        '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
        '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
    ui64* dataw = reinterpret_cast<ui64*>(&data);

    for (unsigned dataItem = 0; dataItem < 4; ++dataItem) {
        __m128i value = _mm_loadu_si128((__m128i*)&dataw[dataItem * 2]);

        ui64 buf[2] = {55, 81};
        _mm_storel_epi64((__m128i*)&buf, value);

        UNIT_ASSERT_EQUAL(dataw[dataItem * 2], buf[0]);
        UNIT_ASSERT_EQUAL(81, buf[1]);
    }
}

void TSSEEmulTest::Test_mm_shuffle_epi32() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    ui32* dataw = reinterpret_cast<ui32*>(&data);
    __m128i value = _mm_loadu_si128((__m128i*)&data);

    int coding[4] = {1, 3, 0, 2};
    __m128i result = _mm_shuffle_epi32(value, _MM_SHUFFLE(2, 0, 3, 1));

    for (unsigned i = 0; i < 4; ++i)
        UNIT_ASSERT_EQUAL(dataw[coding[i]],
                          TQType<uint32x4_t>::As(result)[i]);
}

static int GetHighBitAt(char data, int at) {
    ui8 udata = data & 0x80;
    return int(udata >> 7) << at;
}

void TSSEEmulTest::Test_mm_movemask_epi8() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    __m128i value = _mm_loadu_si128((__m128i*)&data);

    int result = _mm_movemask_epi8(value);
    int verify = 0;
    for (unsigned i = 0; i < 16; ++i) {
        verify |= GetHighBitAt(data[i], i);
    }

    UNIT_ASSERT_EQUAL(result, verify);
}

void TSSEEmulTest::Test_mm_movemask_ps() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    __m128 value = _mm_loadu_ps((float*)&data);

    int result = _mm_movemask_ps(value);
    int verify = 0;
    for (unsigned i = 0; i < 4; ++i) {
        verify |= GetHighBitAt(data[i * 4 + 3], i);
    }

    UNIT_ASSERT_EQUAL(result, verify);
}

void TSSEEmulTest::Test_mm_movemask_ps_2() {
    char data[16] = {
        '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF',
        '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'};
    __m128 value = _mm_loadu_ps((float*)&data);

    int result = _mm_movemask_ps(value);
    UNIT_ASSERT_EQUAL(result, 0xf);
}

void TSSEEmulTest::Test_mm_cvtsi128_si32() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    __m128i value = _mm_loadu_si128((__m128i*)&data);

    int result = _mm_cvtsi128_si32(value);
    i32* datap = reinterpret_cast<i32*>(&data);
    int verify = datap[0];

    UNIT_ASSERT_EQUAL(result, verify);
}

void TSSEEmulTest::Test_mm_cvtsi128_si64() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    __m128i value = _mm_loadu_si128((__m128i*)&data);

    i64 result = _mm_cvtsi128_si64(value);
    i64* datap = reinterpret_cast<i64*>(&data);
    i64 verify = datap[0];

    UNIT_ASSERT_EQUAL(result, verify);
}

void TSSEEmulTest::Test_mm_set_epi16() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    i16* dataw = reinterpret_cast<i16*>(&data);
    ui64* dataq = reinterpret_cast<ui64*>(&data);

    __m128i result = _mm_set_epi16(dataw[7], dataw[6], dataw[5], dataw[4],
                                   dataw[3], dataw[2], dataw[1], dataw[0]);
    ui64 buf[2] = {53, 81};
    _mm_storeu_si128((__m128i*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
    UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
}

void TSSEEmulTest::Test_mm_set_epi32() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    i32* dataw = reinterpret_cast<i32*>(&data);
    ui64* dataq = reinterpret_cast<ui64*>(&data);

    __m128i result = _mm_set_epi32(dataw[3], dataw[2], dataw[1], dataw[0]);
    ui64 buf[2] = {53, 81};
    _mm_storeu_si128((__m128i*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
    UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
}

void TSSEEmulTest::Test_mm_set_ps() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    float* dataw = reinterpret_cast<float*>(&data);
    ui64* dataq = reinterpret_cast<ui64*>(&data);

    __m128 result = _mm_set_ps(dataw[3], dataw[2], dataw[1], dataw[0]);
    ui64 buf[2] = {53, 81};
    _mm_storeu_ps((float*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
    UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
}

void TSSEEmulTest::Test_mm_set_pd() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    double* dataw = reinterpret_cast<double*>(&data);
    ui64* dataq = reinterpret_cast<ui64*>(&data);

    __m128d result = _mm_set_pd(dataw[1], dataw[0]);
    ui64 buf[2] = {53, 81};
    _mm_storeu_pd((double*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
    UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
}

void TSSEEmulTest::Test_mm_cvtsi32_si128() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    i32* dataw = reinterpret_cast<i32*>(&data);

    __m128i result = _mm_cvtsi32_si128(dataw[0]);
    i32 buf[4] = {53, 81, -43, 2132};
    _mm_storeu_si128((__m128i*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataw[0]);
    UNIT_ASSERT_EQUAL(buf[1], 0);
    UNIT_ASSERT_EQUAL(buf[2], 0);
    UNIT_ASSERT_EQUAL(buf[3], 0);
}

void TSSEEmulTest::Test_mm_cvtsi64_si128() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    i64* dataw = reinterpret_cast<i64*>(&data);

    __m128i result = _mm_cvtsi64_si128(dataw[0]);
    i64 buf[2] = {7, 8};
    _mm_storeu_si128((__m128i*)&buf, result);

    UNIT_ASSERT_EQUAL(buf[0], dataw[0]);
    UNIT_ASSERT_EQUAL(buf[1], 0);
}

template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc>
void TSSEEmulTest::Test_mm_packs_epiXX() {
    char data[32] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x00', '\x66', '\x1C',
        '\x99', '\x33', '\x1C', '\x55', '\x00', '\x00', '\x00', '\x00',
        '\x00', '\xAA', '\x00', '\x00', '\xCC', '\xBB', '\x22', '\xFF'};
    __m128i value0 = _mm_loadu_si128((__m128i*)&data);
    __m128i value1 = _mm_loadu_si128(((__m128i*)&data) + 1);
    TElem* dataw = reinterpret_cast<TElem*>(&data);

    __m128i result = TFunc(value0, value1);

    TNarrow verify[elemCount];
    for (unsigned i = 0; i < elemCount; ++i) {
        TElem sum = dataw[i];
        if (sum > std::numeric_limits<TNarrow>::max())
            sum = std::numeric_limits<TNarrow>::max();
        if (sum < std::numeric_limits<TNarrow>::min())
            sum = std::numeric_limits<TNarrow>::min();
        verify[i] = TNarrow(sum);
    }

    ui64* verifyp = (ui64*)&verify;
    UNIT_ASSERT_EQUAL(verifyp[0], TQType<uint64x2_t>::As(result)[0]);
    UNIT_ASSERT_EQUAL(verifyp[1], TQType<uint64x2_t>::As(result)[1]);
}

void TSSEEmulTest::Test_mm_packs_epi16() {
    Test_mm_packs_epiXX<i16, i8, 16, Wrap(_mm_packs_epi16)>();
}
void TSSEEmulTest::Test_mm_packs_epi32() {
    Test_mm_packs_epiXX<i32, i16, 8, Wrap(_mm_packs_epi32)>();
}
void TSSEEmulTest::Test_mm_packus_epi16() {
    Test_mm_packs_epiXX<i16, ui8, 16, Wrap(_mm_packus_epi16)>();
}

void TSSEEmulTest::Test_mm_extract_epi8() {
    alignas(16) char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    const ui8* dataw = reinterpret_cast<const ui8*>(&data);
    const __m128i value = _mm_loadu_si128((__m128i*)&data);

    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 1)), int(dataw[1]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 2)), int(dataw[2]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 3)), int(dataw[3]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 4)), int(dataw[4]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 5)), int(dataw[5]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 6)), int(dataw[6]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 7)), int(dataw[7]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 8)), int(dataw[8]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 9)), int(dataw[9]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 10)), int(dataw[10]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 11)), int(dataw[11]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 12)), int(dataw[12]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 13)), int(dataw[13]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 14)), int(dataw[14]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 15)), int(dataw[15]));
}

void TSSEEmulTest::Test_mm_extract_epi16() {
    alignas(16) char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    const ui16* dataw = reinterpret_cast<const ui16*>(&data);
    const __m128i value = _mm_loadu_si128((__m128i*)&data);

    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 1)), int(dataw[1]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 2)), int(dataw[2]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 3)), int(dataw[3]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 4)), int(dataw[4]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 5)), int(dataw[5]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 6)), int(dataw[6]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 7)), int(dataw[7]));
}

void TSSEEmulTest::Test_mm_extract_epi64() {
    alignas(16) char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    const ui64* dataw = reinterpret_cast<const ui64*>(&data);
    const __m128i value = _mm_loadu_si128((__m128i*)&data);

    UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 0)), (long long)(dataw[0]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 1)), (long long)(dataw[1]));
}

void TSSEEmulTest::Test_mm_extract_epi32() {
    alignas(16) char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    const ui32* dataw = reinterpret_cast<const ui32*>(&data);
    const __m128i value = _mm_loadu_si128((__m128i*)&data);

    UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 0)), int(dataw[0]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 1)), int(dataw[1]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 2)), int(dataw[2]));
    UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 3)), int(dataw[3]));
}

void TSSEEmulTest::Test_MM_TRANSPOSE4_PS() {
    char data0[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    char data1[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
    char data2[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    char data3[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};

    __m128 value0 = _mm_loadu_ps((float*)&data0);
    __m128 value1 = _mm_loadu_ps((float*)&data1);
    __m128 value2 = _mm_loadu_ps((float*)&data2);
    __m128 value3 = _mm_loadu_ps((float*)&data3);

    _MM_TRANSPOSE4_PS(value0, value1, value2, value3);

    ui64 tbuf0[2] = {0, 0};
    ui64 tbuf1[2] = {0, 0};
    ui64 tbuf2[2] = {0, 0};
    ui64 tbuf3[2] = {0, 0};

    _mm_storeu_ps((float*)&tbuf0, value0);
    _mm_storeu_ps((float*)&tbuf1, value1);
    _mm_storeu_ps((float*)&tbuf2, value2);
    _mm_storeu_ps((float*)&tbuf3, value3);

    char tdata0[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55',
        '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55'};
    char tdata1[16] = {
        '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44',
        '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44'};
    char tdata2[16] = {
        '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11',
        '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11'};
    char tdata3[16] = {
        '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF',
        '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF'};

    UNIT_ASSERT(memcmp(tbuf0, tdata0, 16) == 0);
    UNIT_ASSERT(memcmp(tbuf1, tdata1, 16) == 0);
    UNIT_ASSERT(memcmp(tbuf2, tdata2, 16) == 0);
    UNIT_ASSERT(memcmp(tbuf3, tdata3, 16) == 0);
}

template <typename TFrom, typename TTo, unsigned elemCount,
          typename TLoadVector, typename TResultVector,
          typename TElemFunc, typename TFunc, typename TOp>
void TSSEEmulTest::Test_mm_convertop() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    TFrom* datap = reinterpret_cast<TFrom*>(&data);

    TLoadVector value = TFuncLoad<TLoadVector>(&data);

    TTo procData[elemCount];
    for (unsigned i = 0; i < elemCount; ++i) {
        procData[i] = TElemFunc::Call(datap[i]);
    }

    TResultVector result = TFunc(value);

    for (unsigned i = 0; i < elemCount; ++i) {
        UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]);
    }
}

void TSSEEmulTest::Test_mm_cvtepi32_ps() {
    struct THelper {
        static float Call(const i32 op) {
            return float(op);
        }
    };
    Test_mm_convertop<i32, float, 4, __m128i, __m128,
                      THelper, WrapF(_mm_cvtepi32_ps), float32x4_t>();
}

void TSSEEmulTest::Test_mm_cvtps_epi32() {
    struct THelper {
        static i32 Call(const float op) {
            return i32(op);
        }
    };
    Test_mm_convertop<float, i32, 4, __m128, __m128i,
                      THelper, T_mm_CallWrapper<__m128i, decltype(_mm_cvtps_epi32), _mm_cvtps_epi32>, int32x4_t>();
}

void TSSEEmulTest::Test_mm_cvttps_epi32() {
    struct THelper {
        static i32 Call(const float op) {
            return i32(op);
        }
    };
    Test_mm_convertop<float, i32, 4, __m128, __m128i,
                      THelper, Wrap(_mm_cvttps_epi32), int32x4_t>();
}

template <typename TLoadVector, typename TCastVector,
          typename TFunc, TFunc* func>
void TSSEEmulTest::Test_mm_castXX() {
    char data[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};

    TLoadVector value = TFuncLoad<TLoadVector>(&data);
    const TLoadVector constvalue = TFuncLoad<TLoadVector>(&data);
    TCastVector casted = func(value);
    const TCastVector constcasted = func(constvalue);
    char verify[16];
    char constverify[16];
    TFuncStore<TCastVector>(&verify, casted);
    TFuncStore<TCastVector>(&constverify, constcasted);

    UNIT_ASSERT(memcmp(&data, &verify, 16) == 0);
    UNIT_ASSERT(memcmp(&data, &constverify, 16) == 0);
}

void TSSEEmulTest::Test_mm_castsi128_ps() {
    Test_mm_castXX<__m128i, __m128,
                   decltype(_mm_castsi128_ps), _mm_castsi128_ps>();
}

void TSSEEmulTest::Test_mm_castps_si128() {
    Test_mm_castXX<__m128, __m128i,
                   decltype(_mm_castps_si128), _mm_castps_si128>();
}

void TSSEEmulTest::Test_mm_mul_epu32() {
    char data0[16] = {
        '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
        '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
    char data1[16] = {
        '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
        '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
    ui32* dataw0 = reinterpret_cast<ui32*>(&data0);
    ui32* dataw1 = reinterpret_cast<ui32*>(&data1);

    __m128i value0 = _mm_loadu_si128((__m128i*)&data0);
    __m128i value1 = _mm_loadu_si128((__m128i*)&data1);

    ui64 mul0 = (ui64) dataw0[0] * (ui64) dataw1[0];
    ui64 mul1 = (ui64) dataw0[2] * (ui64) dataw1[2];

    __m128i result = _mm_mul_epu32(value0, value1);

    UNIT_ASSERT_EQUAL(mul0, TQType<uint64x2_t>::As(result)[0]);
    UNIT_ASSERT_EQUAL(mul1, TQType<uint64x2_t>::As(result)[1]);
}

void TSSEEmulTest::Test_mm_cmpunord_ps() {
    alignas(16) float valuesBits[4] = {1.f, 2.f, 3.f, 4.f};
    alignas(16) float values2Bits[4] = {5.f, 6.f, 7.f, 8.f};

    alignas(16) char allfs[16] = {
        '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff',
        '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff'
    };

    alignas(16) char allzeroes[16] = {
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
        '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'
    };

    const __m128 qnan = _mm_set_ps1(std::numeric_limits<float>::quiet_NaN());
    const __m128 snan = _mm_set_ps1(std::numeric_limits<float>::signaling_NaN());
    const __m128 values = _mm_loadu_ps((const float*) valuesBits);
    const __m128 values2 = _mm_loadu_ps((const float*) values2Bits);

    const __m128 mask1 = _mm_cmpunord_ps(qnan, qnan);
    UNIT_ASSERT_EQUAL(::memcmp(&mask1, &allfs, sizeof(allfs)), 0);

    const __m128 mask2 = _mm_cmpunord_ps(values, values);
    UNIT_ASSERT_EQUAL(::memcmp(&mask2, &allzeroes, sizeof(allzeroes)), 0);

    const __m128 mask3 = _mm_cmpunord_ps(snan, snan);
    UNIT_ASSERT_EQUAL(::memcmp(&mask3, &allfs, sizeof(allfs)), 0);

    const __m128 mask4 = _mm_cmpunord_ps(qnan, values);
    UNIT_ASSERT_EQUAL(::memcmp(&mask4, &allfs, sizeof(allfs)), 0);

    const __m128 mask5 = _mm_cmpunord_ps(snan, values);
    UNIT_ASSERT_EQUAL(::memcmp(&mask5, &allfs, sizeof(allfs)), 0);

    const __m128 mask6 = _mm_cmpunord_ps(qnan, snan);
    UNIT_ASSERT_EQUAL(::memcmp(&mask6, &allfs, sizeof(allfs)), 0);

    const __m128 mask7 = _mm_cmpunord_ps(values, values2);
    UNIT_ASSERT_EQUAL(::memcmp(&mask7, &allzeroes, sizeof(allzeroes)), 0);
}

void TSSEEmulTest::Test_mm_store_ss() {
    alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f};
    const __m128 value = _mm_loadu_ps(valueBits);
    float res = std::numeric_limits<float>::signaling_NaN();
    _mm_store_ss(&res, value);
    UNIT_ASSERT_EQUAL(res, 1.f);
}

void TSSEEmulTest::Test_mm_store_ps() {
    alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f};
    const __m128 value = _mm_loadu_ps(valueBits);
    float res[4] = {0.f};
    _mm_storeu_ps(res, value);
    UNIT_ASSERT_EQUAL(res[0], 1.f);
    UNIT_ASSERT_EQUAL(res[1], 2.f);
    UNIT_ASSERT_EQUAL(res[2], 3.f);
    UNIT_ASSERT_EQUAL(res[3], 4.f);
}

void TSSEEmulTest::Test_mm_storeu_pd() {
    alignas(16) const double valueBits[4] = {1., 2., 3., 4.};
    for (size_t i = 0; i != 3; ++i) {
        const __m128d value = _mm_loadu_pd(&valueBits[i]);
        alignas(16) double res[4];
        for (size_t shift = 0; shift != 3; ++shift) {
            _mm_storeu_pd(&res[shift], value);
            for (size_t j = 0; j != 2; ++j) {
                UNIT_ASSERT_EQUAL_C(res[j + shift], valueBits[i + j], "res: " << HexEncode(&res[shift], 16) << " vs etalon: " << HexEncode(&valueBits[i], 16));
            }
        }
    }
}

void TSSEEmulTest::Test_mm_andnot_ps() {
    alignas(16) const char firstBits[16] = {
        '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff',
        '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff'
    };

    alignas(16) const char secondBits[16] = {
        '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff',
        '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff'
    };

    alignas(16) const char resBits[16] = {
        '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00',
        '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00'
    };

    const __m128 value1 = _mm_loadu_ps((const float*) firstBits);
    const __m128 value2 = _mm_loadu_ps((const float*) secondBits);
    const __m128 res = _mm_andnot_ps(value1, value2);

    UNIT_ASSERT_EQUAL(::memcmp(&res, resBits, sizeof(resBits)), 0);
}

void TSSEEmulTest::Test_mm_shuffle_ps() {
    alignas(16) const float first[4] = {1.f, 2.f, 3.f, 4.f};
    alignas(16) const float second[4] = {5.f, 6.f, 7.f, 8.f};
    alignas(16) const float etalon[4] = {3.f, 4.f, 5.f, 6.f};

    const __m128 value1 = _mm_loadu_ps(first);
    const __m128 value2 = _mm_loadu_ps(second);
    const __m128 res = _mm_shuffle_ps(value1, value2, _MM_SHUFFLE(1, 0, 3, 2));

    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0);
}

void TSSEEmulTest::Test_mm_shuffle_pd() {
    const double first[2] = {1.3, 2.3};
    const double second[2] = {5.3, 6.3};
    const double etalon0[2] = {1.3, 5.3};
    const double etalon1[2] = {2.3, 5.3};
    const double etalon2[2] = {1.3, 6.3};
    const double etalon3[2] = {2.3, 6.3};

    const __m128d value1 = _mm_loadu_pd(first);
    const __m128d value2 = _mm_loadu_pd(second);

    __m128d res = _mm_shuffle_pd(value1, value2, 0);
    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon0, sizeof(etalon0)), 0);

    res = _mm_shuffle_pd(value1, value2, 1);
    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon1, sizeof(etalon1)), 0);

    res = _mm_shuffle_pd(value1, value2, 2);
    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon2, sizeof(etalon2)), 0);

    res = _mm_shuffle_pd(value1, value2, 3);
    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon3, sizeof(etalon3)), 0);
}

void TSSEEmulTest::Test_mm_cvtsd_f64() {
    const double first[2] = {1.3, 2.3};
    const double second[2] = {5.3, 6.3};

    const __m128d value1 = _mm_loadu_pd(first);
    const __m128d value2 = _mm_loadu_pd(second);

    UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value1), 1.3);
    UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value2), 5.3);
}

void TSSEEmulTest::Test_mm_loadl_pd() {
    const double first[2] = {1.3, 2.3};
    const double second[2] = {5.3, 6.3};
    const double firstEtalon[2] = {10.13, 2.3};
    const double secondEtalon[2] = {11.13, 6.3};

    double newFirst = 10.13;
    double newSecond = 11.13;

    __m128d value1 = _mm_loadu_pd(first);
    __m128d value2 = _mm_loadu_pd(second);
    value1 = _mm_loadl_pd(value1, &newFirst);
    value2 = _mm_loadl_pd(value2, &newSecond);
    UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0);
    UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0);
}

void TSSEEmulTest::Test_mm_loadh_pd() {
    const double first[2] = {1.3, 2.3};
    const double second[2] = {5.3, 6.3};
    const double firstEtalon[2] = {1.3, 10.13};
    const double secondEtalon[2] = {5.3, 11.13};

    double newFirst = 10.13;
    double newSecond = 11.13;

    __m128d value1 = _mm_loadu_pd(first);
    __m128d value2 = _mm_loadu_pd(second);
    value1 = _mm_loadh_pd(value1, &newFirst);
    value2 = _mm_loadh_pd(value2, &newSecond);
    UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0);
    UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0);
}

void TSSEEmulTest::Test_mm_or_ps() {
    alignas(16) const char bytes1[16] = {
        '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff',
        '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff'
    };

    alignas(16) const char bytes2[16] = {
        '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff',
        '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff'
    };

    alignas(16) const char etalon[16] = {
        '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff',
        '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff'
    };

    const __m128 value1 = _mm_loadu_ps((const float*) bytes1);
    const __m128 value2 = _mm_loadu_ps((const float*) bytes2);
    const __m128 res = _mm_or_ps(value1, value2);

    UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0);
}

void TSSEEmulTest::Test_mm_loadu_pd() {
    alignas(16) double stub[4] = {
        0.f, 1.f,
        2.f, 3.f
    };

    for (size_t shift = 0; shift != 3; ++shift) {
        const __m128d val = _mm_loadu_pd(&stub[shift]);
        alignas(16) double res[2];
        _mm_store_pd(res, val);

        for (size_t i = 0; i != 2; ++i) {
            UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
        }
    }
}

void TSSEEmulTest::Test_mm_rsqrt_ps() {
    alignas(16) const char bytes[16] = {
        '\x00', '\x00', '\x28', '\x42', // 42.f
        '\x00', '\x98', '\x84', '\x45', // 4243.f
        '\x60', '\x26', '\xcf', '\x48', // 424243.f
        '\xed', '\xd5', '\x21', '\x4c'  // 42424243.f
    };
    const __m128 value = _mm_loadu_ps((const float*)bytes);
    const __m128 result = _mm_rsqrt_ps(value);
    alignas(16) float res[4];
    _mm_store_ps(res, result);
    float fResult = 0.f;
    for (size_t i = 0; i < 4; ++i) {
        memcpy(&fResult, &bytes[i * 4], 4);
        fResult = 1.f / std::sqrt(fResult);
        UNIT_ASSERT_DOUBLES_EQUAL_C(res[i], fResult, 1e-3, "res: " << fResult << " vs etalon " << res[i]);
    }
}

namespace NHelpers {

    static __m128i Y_FORCE_INLINE GetCmp16(const __m128 &c0, const __m128 &c1, const __m128 &c2, const __m128 &c3, const __m128 test) {
        const __m128i r0 = _mm_castps_si128(_mm_cmpgt_ps(c0, test));
        const __m128i r1 = _mm_castps_si128(_mm_cmpgt_ps(c1, test));
        const __m128i r2 = _mm_castps_si128(_mm_cmpgt_ps(c2, test));
        const __m128i r3 = _mm_castps_si128(_mm_cmpgt_ps(c3, test));
        const __m128i packed = _mm_packs_epi16(_mm_packs_epi32(r0, r1), _mm_packs_epi32(r2, r3));
        return _mm_and_si128(_mm_set1_epi8(0x01), packed);
    }

    static __m128i Y_FORCE_INLINE GetCmp16(const float *factors, const __m128 test) {
        const __m128 *ptr = (__m128 *)factors;
        return GetCmp16(ptr[0], ptr[1], ptr[2], ptr[3], test);
    }

    template<size_t Num>
    void DoLane(size_t length, const float *factors, ui32 *& dst, const float *&values) {
        for (size_t i = 0; i < length; ++i) {
            __m128 value = _mm_set1_ps(values[i]);
            __m128i agg = GetCmp16(factors, value);
            if (Num > 1) {
                agg = _mm_add_epi16(agg, _mm_slli_epi16(GetCmp16(&factors[64], value), 1));
            }
            _mm_store_si128((__m128i *)&dst[4 * i], agg);
        }
    }
}

void TSSEEmulTest::Test_matrixnet_powerpc() {
    static constexpr size_t length = 10;
    alignas(16) float factors[1024];
    alignas(16) ui32 valP[4 * length] = { 0 };
    float values[length];
    TReallyFastRng32 rng(42);
    for (size_t i = 0; i < 1024; ++i) {
        factors[i] = rng.GenRandReal2();
    }
    for (size_t i = 0; i < length; ++i) {
        values[i] = rng.GenRandReal2();
    }
    ui32* val = reinterpret_cast<ui32*>(valP);
    const float* vals = reinterpret_cast<const float*>(values);
    NHelpers::DoLane<2>(length, factors, val, vals);
    static const ui32 etalon[4 * length] = {
        2, 33554432, 258, 33554433, 50529027,
        50529027, 50529027, 50529027, 50528770,
        33685763, 33555203, 50462723, 50528770,
        33685763, 33555203, 50462723, 50529026,
        33751299, 50529027, 50463491, 2, 33554432,
        258, 33554433, 50397698, 33685761, 259,
        50462721, 50332162, 33554689, 259, 50462721,
        50528770, 33685761, 33555203, 50462723,
        50529026, 33685763, 50463491, 50463235
    };
    for (size_t i = 0; i < 4 * length; ++i) {
        UNIT_ASSERT_EQUAL(valP[i], etalon[i]);
    }
}