stdlib/stdlib.ispc - main - redgpu/ispc

stdlib/stdlib.ispc
8 132 строки338 KB
Jérôme Richard
Implement the function log1p
11 май 2026, 06:54
11 май 2026, 06:54f940c96
// -*- mode: c++ -*-
/*
  Copyright (c) 2010-2026, Intel Corporation

  SPDX-License-Identifier: BSD-3-Clause
*/

/** @file stdlib.ispc

    @brief Portion of the ispc standard library implementation that's in
           ispc code
*/

#define ISPC_INTERNAL_STDLIB_COMPILATION
#include "core.isph"

#include "builtins.isph"
#include "stdlib.isph"
#include "svml.isph"
#include "target.isph"
#undef ISPC_INTERNAL_STDLIB_COMPILATION

///////////////////////////////////////////////////////////////////////////
// Low level primitives

__declspec(safe, cost0) static inline float16 float16bits(unsigned int16 a) { return __halfbits_varying_int16(a); }

__declspec(safe, cost0) static inline uniform float16 float16bits(uniform unsigned int16 a) {
    return __halfbits_uniform_int16(a);
}

__declspec(safe, cost0) static inline float16 float16bits(int16 a) { return __halfbits_varying_int16(a); }

__declspec(safe, cost0) static inline uniform float16 float16bits(uniform int16 a) {
    return __halfbits_uniform_int16(a);
}

__declspec(safe, cost0) static inline float floatbits(unsigned int a) { return __floatbits_varying_int32(a); }

__declspec(safe, cost0) static inline uniform float floatbits(uniform unsigned int a) {
    return __floatbits_uniform_int32(a);
}

__declspec(safe, cost0) static inline float floatbits(int a) { return __floatbits_varying_int32(a); }

__declspec(safe, cost0) static inline uniform float floatbits(uniform int a) { return __floatbits_uniform_int32(a); }

__declspec(safe, cost0) static inline double doublebits(unsigned int64 a) { return __doublebits_varying_int64(a); }

__declspec(safe, cost0) static inline uniform double doublebits(uniform unsigned int64 a) {
    return __doublebits_uniform_int64(a);
}

__declspec(safe, cost0) static inline unsigned int16 intbits(float16 a) { return __intbits_varying_half(a); }

__declspec(safe, cost0) static inline uniform unsigned int16 intbits(uniform float16 a) {
    return __intbits_uniform_half(a);
}

__declspec(safe, cost0) static inline unsigned int intbits(float a) { return __intbits_varying_float(a); }

__declspec(safe, cost0) static inline uniform unsigned int intbits(uniform float a) {
    return __intbits_uniform_float(a);
}

__declspec(safe, cost0) static inline unsigned int64 intbits(double d) { return __intbits_varying_double(d); }

__declspec(safe, cost0) static inline uniform unsigned int64 intbits(uniform double d) {
    return __intbits_uniform_double(d);
}

__declspec(safe) static inline float broadcast(float v, uniform int i) { return __broadcast_float(v, i); }

__declspec(safe) static inline int8 broadcast(int8 v, uniform int i) { return __broadcast_i8(v, i); }

__declspec(safe) static inline unsigned int8 broadcast(unsigned int8 v, uniform int i) { return __broadcast_i8(v, i); }

__declspec(safe) static inline int16 broadcast(int16 v, uniform int i) { return __broadcast_i16(v, i); }

__declspec(safe) static inline unsigned int16 broadcast(unsigned int16 v, uniform int i) {
    return __broadcast_i16(v, i);
}

__declspec(safe) static inline float16 broadcast(float16 v, uniform int i) { return __broadcast_half(v, i); }

__declspec(safe) static inline int32 broadcast(int32 v, uniform int i) { return __broadcast_i32(v, i); }

__declspec(safe) static inline unsigned int32 broadcast(unsigned int32 v, uniform int i) {
    return __broadcast_i32(v, i);
}

__declspec(safe) static inline double broadcast(double v, uniform int i) { return __broadcast_double(v, i); }

__declspec(safe) static inline int64 broadcast(int64 v, uniform int i) { return __broadcast_i64(v, i); }

__declspec(safe) static inline unsigned int64 broadcast(unsigned int64 v, uniform int i) {
    return __broadcast_i64(v, i);
}

template <typename T> unmasked varying T __rotate_via_shuffle(varying T v, uniform int32 i) {
    // Pre-computed shuffle masks for each rotation
#if TARGET_WIDTH == 4
    static const int masks[4] = {
        {0, 1, 2, 3}, // rotate by 0
        {1, 2, 3, 0}, // rotate by 1
        {2, 3, 0, 1}, // rotate by 2
        {3, 0, 1, 2}  // rotate by 3
    };
#elif TARGET_WIDTH == 8
    static const int masks[8] = {
        {0, 1, 2, 3, 4, 5, 6, 7}, // rotate by 0
        {1, 2, 3, 4, 5, 6, 7, 0}, // rotate by 1
        {2, 3, 4, 5, 6, 7, 0, 1}, // rotate by 2
        {3, 4, 5, 6, 7, 0, 1, 2}, // rotate by 3
        {4, 5, 6, 7, 0, 1, 2, 3}, // rotate by 4
        {5, 6, 7, 0, 1, 2, 3, 4}, // rotate by 5
        {6, 7, 0, 1, 2, 3, 4, 5}, // rotate by 6
        {7, 0, 1, 2, 3, 4, 5, 6}  // rotate by 7
    };
#elif TARGET_WIDTH == 16
    static const int masks[16] = {
        {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, // rotate by 0
        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0}, // rotate by 1
        {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1}, // rotate by 2
        {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2}, // rotate by 3
        {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3}, // rotate by 4
        {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4}, // rotate by 5
        {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5}, // rotate by 6
        {7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6}, // rotate by 7
        {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}, // rotate by 8
        {9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8}, // rotate by 9
        {10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // rotate by 10
        {11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // rotate by 11
        {12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, // rotate by 12
        {13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, // rotate by 13
        {14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, // rotate by 14
        {15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}  // rotate by 15
    };
#elif TARGET_WIDTH == 32
    static const int masks[32] = {
        {0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, // rotate by 0
        {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16,
         17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0}, // rotate by 1
        {2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17,
         18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1}, // rotate by 2
        {3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18,
         19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2}, // rotate by 3
        {4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
         20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3}, // rotate by 4
        {5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
         21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4}, // rotate by 5
        {6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
         22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5}, // rotate by 6
        {7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
         23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6}, // rotate by 7
        {8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
         24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7}, // rotate by 8
        {9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
         25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8}, // rotate by 9
        {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
         26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9}, // rotate by 10
        {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
         27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10}, // rotate by 11
        {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
         28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11}, // rotate by 12
        {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
         29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12}, // rotate by 13
        {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
         30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13}, // rotate by 14
        {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
         31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14}, // rotate by 15
        {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
         0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15}, // rotate by 16
        {17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,
         1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16}, // rotate by 17
        {18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,
         2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17}, // rotate by 18
        {19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,
         3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18}, // rotate by 19
        {20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,
         4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, // rotate by 20
        {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,
         5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, // rotate by 21
        {22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,
         6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}, // rotate by 22
        {23, 24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,
         7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}, // rotate by 23
        {24, 25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,
         8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}, // rotate by 24
        {25, 26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,
         9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}, // rotate by 25
        {26, 27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
         10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}, // rotate by 26
        {27, 28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10,
         11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26}, // rotate by 27
        {28, 29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11,
         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}, // rotate by 28
        {29, 30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
         13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}, // rotate by 29
        {30, 31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13,
         14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}, // rotate by 30
        {31, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14,
         15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30} // rotate by 31
    };
#elif TARGET_WIDTH == 64
    static const int masks[64] = {
        {0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
         22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43,
         44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}, // rotate by 0
        {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
         23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
         45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0}, // rotate by 1
        {2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
         24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,
         46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1}, // rotate by 2
        {3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
         25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46,
         47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2}, // rotate by 3
        {4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
         26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
         48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3}, // rotate by 4
        {5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
         27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
         49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4}, // rotate by 5
        {6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
         28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
         50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5}, // rotate by 6
        {7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
         29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
         51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6}, // rotate by 7
        {8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
         30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
         52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7}, // rotate by 8
        {9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
         31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
         53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8}, // rotate by 9
        {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
         32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
         54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9}, // rotate by 10
        {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
         33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
         55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10}, // rotate by 11
        {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
         34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
         56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11}, // rotate by 12
        {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
         35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,
         57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12}, // rotate by 13
        {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
         36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
         58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13}, // rotate by 14
        {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
         37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
         59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14}, // rotate by 15
        {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
         38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
         60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15}, // rotate by 16
        {17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38,
         39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,
         61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16}, // rotate by 17
        {18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
         40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
         62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17}, // rotate by 18
        {19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
         41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62,
         63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18}, // rotate by 19
        {20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
         42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
         0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, // rotate by 20
        {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42,
         43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,
         1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, // rotate by 21
        {22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43,
         44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,
         2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}, // rotate by 22
        {23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
         45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,
         3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}, // rotate by 23
        {24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,
         46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,
         4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}, // rotate by 24
        {25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46,
         47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,
         5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}, // rotate by 25
        {26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
         48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,
         6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}, // rotate by 26
        {27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
         49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,
         7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26}, // rotate by 27
        {28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
         50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,
         8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}, // rotate by 28
        {29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
         51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,
         9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}, // rotate by 29
        {30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
         52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
         10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}, // rotate by 30
        {31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
         53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10,
         11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}, // rotate by 31
        {32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
         54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11,
         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, // rotate by 32
        {33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
         55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
         13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}, // rotate by 33
        {34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
         56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13,
         14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}, // rotate by 34
        {35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,
         57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14,
         15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, // rotate by 35
        {36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
         58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35}, // rotate by 36
        {37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
         59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16,
         17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36}, // rotate by 37
        {38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
         60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17,
         18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37}, // rotate by 38
        {39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,
         61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18,
         19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38}, // rotate by 39
        {40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
         62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
         20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}, // rotate by 40
        {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62,
         63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
         21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40}, // rotate by 41
        {42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
         0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
         22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41}, // rotate by 42
        {43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,
         1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
         23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42}, // rotate by 43
        {44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,
         2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
         24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43}, // rotate by 44
        {45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,
         3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
         25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44}, // rotate by 45
        {46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,
         4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
         26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45}, // rotate by 46
        {47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,
         5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
         27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46}, // rotate by 47
        {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,
         6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
         28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}, // rotate by 48
        {49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,
         7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
         29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48}, // rotate by 49
        {50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,
         8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
         30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49}, // rotate by 50
        {51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,
         9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
         31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50}, // rotate by 51
        {52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
         10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
         32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51}, // rotate by 52
        {53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10,
         11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
         33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52}, // rotate by 53
        {54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11,
         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
         34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53}, // rotate by 54
        {55, 56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
         13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
         35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54}, // rotate by 55
        {56, 57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13,
         14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
         36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55}, // rotate by 56
        {57, 58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14,
         15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
         37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56}, // rotate by 57
        {58, 59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15,
         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
         38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57}, // rotate by 58
        {59, 60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16,
         17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38,
         39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58}, // rotate by 59
        {60, 61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17,
         18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
         40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59}, // rotate by 60
        {61, 62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18,
         19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
         41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}, // rotate by 61
        {62, 63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
         20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
         42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61}, // rotate by 62
        {63, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
         21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42,
         43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62} // rotate by 63
    };
#endif

    i = i & (TARGET_WIDTH - 1);
    return shuffle(v, masks[i]);
}

__declspec(safe) static inline int8 rotate(int8 v, uniform int i) {
    if (__have_rotate_via_shuffle_8) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i8(v, i);
    }
}

__declspec(safe) static inline unsigned int8 rotate(unsigned int8 v, uniform int i) {
    if (__have_rotate_via_shuffle_8) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i8(v, i);
    }
}

__declspec(safe) static inline float16 rotate(float16 v, uniform int i) {
    if (__have_rotate_via_shuffle_16) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_half(v, i);
    }
}

__declspec(safe) static inline int16 rotate(int16 v, uniform int i) {
    if (__have_rotate_via_shuffle_16) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i16(v, i);
    }
}

__declspec(safe) static inline unsigned int16 rotate(unsigned int16 v, uniform int i) {
    if (__have_rotate_via_shuffle_16) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i16(v, i);
    }
}

__declspec(safe) static inline float rotate(float v, uniform int i) {
    if (__have_rotate_via_shuffle_32) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_float(v, i);
    }
}

__declspec(safe) static inline int32 rotate(int32 v, uniform int i) {
    if (__have_rotate_via_shuffle_32) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i32(v, i);
    }
}

__declspec(safe) static inline unsigned int32 rotate(unsigned int32 v, uniform int i) {
    if (__have_rotate_via_shuffle_32) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i32(v, i);
    }
}

__declspec(safe) static inline double rotate(double v, uniform int i) {
    if (__have_rotate_via_shuffle_64) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_double(v, i);
    }
}

__declspec(safe) static inline int64 rotate(int64 v, uniform int i) {
    if (__have_rotate_via_shuffle_64) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i64(v, i);
    }
}

__declspec(safe) static inline unsigned int64 rotate(unsigned int64 v, uniform int i) {
    if (__have_rotate_via_shuffle_64) {
        return __rotate_via_shuffle(v, i);
    } else {
        return __rotate_i64(v, i);
    }
}

template <typename T> unmasked varying T __shift_via_shuffle(varying T v, uniform int32 i) {
    // Unlike circular shift (rotate), this is a linear shift that nullifies
    // out-of-bounds elements (not wrapping around).
    // Calling here shuffle with sourceIndex containing the out-of-bounds
    // indexes is fine until shuffle is implemented with on-registers shuffles.
    // When it is implemented via gather/scatter, it really should not be used,
    // so we don't defined USE_SHIFT_VIA_SHUFFLE_64 for avx512-x32/x64 and
    // USE_SHIFT_VIA_SHUFFLE_32 for avx512-x64.
    varying int32 sourceIndex = programIndex + i;
    varying T shuffled = shuffle(v, sourceIndex);
    varying bool inBounds = (sourceIndex >= 0) & (sourceIndex < TARGET_WIDTH);
    return select(inBounds, shuffled, (varying T)0);
}

__declspec(safe) static inline float shift(float v, uniform int i) {
    if (__have_shift_via_shuffle_32) {
        return __shift_via_shuffle(v, i);
    } else {
        varying float result;
        unmasked { result = __shift_float(v, i); }
        return result;
    }
}

__declspec(safe) static inline int8 shift(int8 v, uniform int i) {
    if (__have_shift_via_shuffle_8) {
        return __shift_via_shuffle(v, i);
    } else {
        varying int8 result;
        unmasked { result = __shift_i8(v, i); }
        return result;
    }
}

__declspec(safe) static inline unsigned int8 shift(unsigned int8 v, uniform int i) {
    if (__have_shift_via_shuffle_8) {
        return __shift_via_shuffle(v, i);
    } else {
        varying unsigned int8 result;
        unmasked { result = __shift_i8(v, i); }
        return result;
    }
}

__declspec(safe) static inline int16 shift(int16 v, uniform int i) {
    if (__have_shift_via_shuffle_16) {
        return __shift_via_shuffle(v, i);
    } else {
        varying int16 result;
        unmasked { result = __shift_i16(v, i); }
        return result;
    }
}

__declspec(safe) static inline unsigned int16 shift(unsigned int16 v, uniform int i) {
    if (__have_shift_via_shuffle_16) {
        return __shift_via_shuffle(v, i);
    } else {
        varying unsigned int16 result;
        unmasked { result = __shift_i16(v, i); }
        return result;
    }
}

__declspec(safe) static inline float16 shift(float16 v, uniform int i) {
    if (__have_shift_via_shuffle_16) {
        return __shift_via_shuffle(v, i);
    } else {
        varying float16 result;
        unmasked { result = __shift_half(v, i); }
        return result;
    }
}

__declspec(safe) static inline int32 shift(int32 v, uniform int i) {
    if (__have_shift_via_shuffle_32) {
        return __shift_via_shuffle(v, i);
    } else {
        varying int32 result;
        unmasked { result = __shift_i32(v, i); }
        return result;
    }
}

__declspec(safe) static inline unsigned int32 shift(unsigned int32 v, uniform int i) {
    if (__have_shift_via_shuffle_32) {
        return __shift_via_shuffle(v, i);
    } else {
        varying unsigned int32 result;
        unmasked { result = __shift_i32(v, i); }
        return result;
    }
}

__declspec(safe) static inline double shift(double v, uniform int i) {
    if (__have_shift_via_shuffle_64) {
        return __shift_via_shuffle(v, i);
    } else {
        varying double result;
        unmasked { result = __shift_double(v, i); }
        return result;
    }
}

__declspec(safe) static inline int64 shift(int64 v, uniform int i) {
    if (__have_shift_via_shuffle_64) {
        return __shift_via_shuffle(v, i);
    } else {
        varying int64 result;
        unmasked { result = __shift_i64(v, i); }
        return result;
    }
}

__declspec(safe) static inline unsigned int64 shift(unsigned int64 v, uniform int i) {
    if (__have_shift_via_shuffle_64) {
        return __shift_via_shuffle(v, i);
    } else {
        varying unsigned int64 result;
        unmasked { result = __shift_i64(v, i); }
        return result;
    }
}

__declspec(safe) static inline float shuffle(float v, int i) { return __shuffle_float(v, i); }

__declspec(safe) static inline int8 shuffle(int8 v, int i) { return __shuffle_i8(v, i); }

__declspec(safe) static inline unsigned int8 shuffle(unsigned int8 v, int i) { return __shuffle_i8(v, i); }

__declspec(safe) static inline int16 shuffle(int16 v, int i) { return __shuffle_i16(v, i); }

__declspec(safe) static inline unsigned int16 shuffle(unsigned int16 v, int i) { return __shuffle_i16(v, i); }

__declspec(safe) static inline float16 shuffle(float16 v, int i) { return __shuffle_half(v, i); }

__declspec(safe) static inline int32 shuffle(int32 v, int i) { return __shuffle_i32(v, i); }

__declspec(safe) static inline unsigned int32 shuffle(unsigned int32 v, int i) { return __shuffle_i32(v, i); }

__declspec(safe) static inline double shuffle(double v, int i) { return __shuffle_double(v, i); }

__declspec(safe) static inline int64 shuffle(int64 v, int i) { return __shuffle_i64(v, i); }

__declspec(safe) static inline unsigned int64 shuffle(unsigned int64 v, int i) { return __shuffle_i64(v, i); }

__declspec(safe) static inline float shuffle(float v0, float v1, int i) { return __shuffle2_float(v0, v1, i); }

__declspec(safe) static inline int8 shuffle(int8 v0, int8 v1, int i) { return __shuffle2_i8(v0, v1, i); }

__declspec(safe) static inline unsigned int8 shuffle(unsigned int8 v0, unsigned int8 v1, int i) {
    return __shuffle2_i8(v0, v1, i);
}

__declspec(safe) static inline int16 shuffle(int16 v0, int16 v1, int i) { return __shuffle2_i16(v0, v1, i); }

__declspec(safe) static inline unsigned int16 shuffle(unsigned int16 v0, unsigned int16 v1, int i) {
    return __shuffle2_i16(v0, v1, i);
}

__declspec(safe) static inline float16 shuffle(float16 v0, float16 v1, int i) { return __shuffle2_half(v0, v1, i); }

__declspec(safe) static inline int32 shuffle(int32 v0, int32 v1, int i) { return __shuffle2_i32(v0, v1, i); }

__declspec(safe) static inline unsigned int32 shuffle(unsigned int32 v0, unsigned int32 v1, int i) {
    return __shuffle2_i32(v0, v1, i);
}

__declspec(safe) static inline double shuffle(double v0, double v1, int i) { return __shuffle2_double(v0, v1, i); }

__declspec(safe) static inline int64 shuffle(int64 v0, int64 v1, int i) { return __shuffle2_i64(v0, v1, i); }

__declspec(safe) static inline unsigned int64 shuffle(unsigned int64 v0, unsigned int64 v1, int i) {
    return __shuffle2_i64(v0, v1, i);
}

// x[i]
__declspec(safe, cost1) static inline uniform float extract(float x, uniform int i) {
    return floatbits(__extract_int32((int)intbits(x), i));
}

__declspec(safe, cost1) static inline uniform bool extract(bool x, uniform int i) { return __extract_bool(x, i); }

__declspec(safe, cost1) static inline uniform int8 extract(int8 x, uniform int i) { return __extract_int8(x, i); }

__declspec(safe, cost1) static inline uniform unsigned int8 extract(unsigned int8 x, uniform int i) {
    return __extract_int8(x, (uniform unsigned int)i);
}

__declspec(safe, cost1) static inline uniform int16 extract(int16 x, uniform int i) { return __extract_int16(x, i); }

__declspec(safe, cost1) static inline uniform unsigned int16 extract(unsigned int16 x, uniform int i) {
    return __extract_int16(x, (uniform unsigned int)i);
}

__declspec(safe, cost1) static inline uniform float16 extract(float16 x, uniform int i) {
    return float16bits(__extract_int16((int16)intbits(x), i));
}

__declspec(safe, cost1) static inline uniform int32 extract(int32 x, uniform int i) { return __extract_int32(x, i); }

__declspec(safe, cost1) static inline uniform unsigned int32 extract(unsigned int32 x, uniform int i) {
    return __extract_int32(x, (uniform unsigned int)i);
}

__declspec(safe, cost1) static inline uniform double extract(double x, uniform int i) {
    return doublebits(__extract_int64((int64)intbits(x), i));
}

__declspec(safe, cost1) static inline uniform int64 extract(int64 x, uniform int i) { return __extract_int64(x, i); }

__declspec(safe, cost1) static inline uniform unsigned int64 extract(unsigned int64 x, uniform int i) {
    return __extract_int64(x, (uniform unsigned int)i);
}
// x[i] = v
__declspec(safe, cost1) static inline float insert(float x, uniform int i, uniform float v) {
    return floatbits(__insert_int32((int)intbits(x), i, (uniform int)intbits(v)));
}

__declspec(safe, cost1) static inline bool insert(bool x, uniform int i, uniform bool v) {
    return __insert_bool(x, i, v);
}

__declspec(safe, cost1) static inline int8 insert(int8 x, uniform int i, uniform int8 v) {
    return __insert_int8(x, i, v);
}

__declspec(safe, cost1) static inline unsigned int8 insert(unsigned int8 x, uniform int i, uniform unsigned int8 v) {
    return __insert_int8(x, (uniform unsigned int)i, v);
}

__declspec(safe, cost1) static inline float16 insert(float16 x, uniform int i, uniform float16 v) {
    return float16bits(__insert_int16((int16)intbits(x), i, (uniform int16)intbits(v)));
}

__declspec(safe, cost1) static inline int16 insert(int16 x, uniform int i, uniform int16 v) {
    return __insert_int16(x, i, v);
}

__declspec(safe, cost1) static inline unsigned int16 insert(unsigned int16 x, uniform int i, uniform unsigned int16 v) {
    return __insert_int16(x, (uniform unsigned int)i, v);
}

__declspec(safe, cost1) static inline int32 insert(int32 x, uniform int i, uniform int32 v) {
    return __insert_int32(x, i, v);
}

__declspec(safe, cost1) static inline unsigned int32 insert(unsigned int32 x, uniform int i, uniform unsigned int32 v) {
    return __insert_int32(x, (uniform unsigned int)i, v);
}

__declspec(safe, cost1) static inline double insert(double x, uniform int i, uniform double v) {
    return doublebits(__insert_int64((int64)intbits(x), i, (uniform int64)intbits(v)));
}

__declspec(safe, cost1) static inline int64 insert(int64 x, uniform int i, uniform int64 v) {
    return __insert_int64(x, i, v);
}

__declspec(safe, cost1) static inline unsigned int64 insert(unsigned int64 x, uniform int i, uniform unsigned int64 v) {
    return __insert_int64(x, (uniform unsigned int)i, v);
}

__declspec(safe, cost1) static inline uniform int32 sign_extend(uniform bool v) { return __sext_uniform_bool(v); }

__declspec(safe, cost1) static inline int32 sign_extend(bool v) { return __sext_varying_bool(v); }

__declspec(safe) static inline uniform bool any(bool v) {
    // We only care about whether "any" is true for the active program instances,
    // so we have to make v with the current program mask.
#if (ISPC_MASK_BITS == 1)
    return __any(v & __mask);
#else
    return __any((UIntMaskType)__sext_varying_bool(v) & __mask);
#endif
}

__declspec(safe) static inline uniform bool all(bool v) {
    // As with any(), we need to explicitly mask v with the current program mask
    // so we're only looking at the current lanes
#if (ISPC_MASK_BITS == 1)
    return __all(v | !__mask);
#else
    // !__mask returns a 'bool' type. But for the logic to work, we need to
    // convert this to a 'UIntMaskType' type. For 'TRUE' bool, we can be
    // certain that the LSB will be set to '1'. Therefore using
    // '__sext_varying_bool(!__mask)' to convert '!__mask' to 'UIntMaskType'
    // is the safest option to ensure all bits are set to '1' for 'TRUE' and
    // '0' for 'False'.
    return __all((UIntMaskType)__sext_varying_bool(v) | (UIntMaskType)__sext_varying_bool(!__mask));
#endif
}

__declspec(safe) static inline uniform bool none(bool v) {
    // As with any(), we need to explicitly mask v with the current program mask
    // so we're only looking at the current lanes
#if (ISPC_MASK_BITS == 1)
    return __none(v & __mask);
#else
    return __none((UIntMaskType)__sext_varying_bool(v) & __mask);
#endif
}

__declspec(safe) static inline uniform int32 popcnt(uniform int32 v) { return __popcnt_int32_uniform(v); }

__declspec(safe) static inline uniform int32 popcnt(uniform int64 v) { return __popcnt_int64_uniform(v); }

__declspec(safe) static inline varying int32 popcnt(varying int32 v) { return __popcnt_int32_varying(v, __mask); }

__declspec(safe) static inline varying int32 popcnt(varying int64 v) { return __popcnt_int64_varying(v, __mask); }

__declspec(safe) static inline uniform int popcnt(bool v) {
    // As with any() and all(), only count across the active lanes
#if (ISPC_MASK_BITS == 1)
    return __popcnt_int64_uniform(__movmsk(v & __mask));
#else
    return __popcnt_int64_uniform(__movmsk((UIntMaskType)__sext_varying_bool(v) & __mask));
#endif
}

__declspec(safe) static inline uniform unsigned int64 lanemask() { return __movmsk(__mask); }

__declspec(safe) static inline uniform unsigned int64 packmask(bool v) {
#if (ISPC_MASK_BITS == 1)
    return __movmsk(v & __mask);
#else
    return __movmsk((UIntMaskType)__sext_varying_bool(v) & __mask);
#endif
}

///////////////////////////////////////////////////////////////////////////
// memcpy/memmove/memset

static inline void memcpy(void *uniform dst, void *uniform src, uniform int32 count) {
    if (__is_xe_target) {
        for (uniform int j = 0; j < count; j++) {
            ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
        }
    } else {
        __memcpy32((uniform int8 * uniform) dst, (uniform int8 * uniform) src, count);
    }
}

static inline void memcpy64(void *uniform dst, void *uniform src, uniform int64 count) {
    if (__is_xe_target) {
        for (uniform int64 j = 0; j < count; j++) {
            ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
        }
    } else {
        __memcpy64((uniform int8 * uniform) dst, (uniform int8 * uniform) src, count);
    }
}

static inline void memcpy(void *varying dst, void *varying src, int32 count) {
    void *uniform da[programCount];
    void *uniform sa[programCount];

    da[programIndex] = dst;
    sa[programIndex] = src;

    foreach_active(i) {
        void *uniform d = da[i], *uniform s = sa[i];
        if (__is_xe_target) {
            for (uniform int j = 0; j < extract(count, i); j++) {
                ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
            }
        } else {
            __memcpy32((uniform int8 * uniform) d, (uniform int8 * uniform) s, extract(count, i));
        }
    }
}

static inline void memcpy64(void *varying dst, void *varying src, int64 count) {
    void *uniform da[programCount];
    void *uniform sa[programCount];

    da[programIndex] = dst;
    sa[programIndex] = src;

    foreach_active(i) {
        void *uniform d = da[i], *uniform s = sa[i];
        if (__is_xe_target) {
            for (uniform int64 j = 0; j < extract(count, i); j++) {
                ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
            }
        } else {
            __memcpy64((uniform int8 * uniform) d, (uniform int8 * uniform) s, extract(count, i));
        }
    }
}

static inline void memmove(void *uniform dst, void *uniform src, uniform int32 count) {
    if (__is_xe_target) {
        if ((uintptr_t)dst - (uintptr_t)src >= (size_t)count) {
            for (uniform int j = 0; j < count; j++) {
                ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
            }
        } else {
            for (uniform int j = count - 1; j >= 0; j--) {
                ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
            }
        }
    } else {
        __memmove32((uniform int8 * uniform) dst, (uniform int8 * uniform) src, count);
    }
}

static inline void memmove64(void *uniform dst, void *uniform src, uniform int64 count) {
    if (__is_xe_target) {
        if ((uintptr_t)dst - (uintptr_t)src >= (size_t)count) {
            for (uniform int64 j = 0; j < count; j++) {
                ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
            }
        } else {
            for (uniform int64 j = count - 1; j >= 0; j--) {
                ((int8 * uniform) dst)[j] = ((int8 * uniform) src)[j];
            }
        }
    } else {
        __memmove64((uniform int8 * uniform) dst, (uniform int8 * uniform) src, count);
    }
}

static inline void memmove(void *varying dst, void *varying src, int32 count) {
    void *uniform da[programCount];
    void *uniform sa[programCount];

    da[programIndex] = dst;
    sa[programIndex] = src;

    foreach_active(i) {
        void *uniform d = da[i], *uniform s = sa[i];
        uniform int c = extract(count, i);
        if (__is_xe_target) {
            if ((uintptr_t)d - (uintptr_t)s >= (size_t)c) {
                for (uniform int j = 0; j < count; j++) {
                    ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
                }
            } else {
                for (uniform int j = c - 1; j >= 0; j--) {
                    ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
                }
            }
        } else {
            __memmove32((uniform int8 * uniform) d, (uniform int8 * uniform) s, c);
        }
    }
}

static inline void memmove64(void *varying dst, void *varying src, int64 count) {
    void *uniform da[programCount];
    void *uniform sa[programCount];

    da[programIndex] = dst;
    sa[programIndex] = src;

    foreach_active(i) {
        void *uniform d = da[i], *uniform s = sa[i];
        uniform int64 c = extract(count, i);
        if (__is_xe_target) {
            if ((uintptr_t)d - (uintptr_t)s >= (size_t)c) {
                for (uniform int64 j = 0; j < count; j++) {
                    ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
                }
            } else {
                for (uniform int64 j = c - 1; j >= 0; j--) {
                    ((int8 * uniform) d)[j] = ((int8 * uniform) s)[j];
                }
            }
        } else {
            __memmove64((uniform int8 * uniform) d, (uniform int8 * uniform) s, c);
        }
    }
}

static inline void memset(void *uniform ptr, uniform int8 val, uniform int32 count) {
    if (__is_xe_target) {
        for (uniform int j = 0; j < count; j++) {
            ((int8 * uniform) ptr)[j] = val;
        }
    } else {
        __memset32((uniform int8 * uniform) ptr, val, count);
    }
}

static inline void memset64(void *uniform ptr, uniform int8 val, uniform int64 count) {
    if (__is_xe_target) {
        for (uniform int64 j = 0; j < count; j++) {
            ((int8 * uniform) ptr)[j] = val;
        }
    } else {
        __memset64((uniform int8 * uniform) ptr, val, count);
    }
}

static inline void memset(void *varying ptr, int8 val, int32 count) {
    void *uniform pa[programCount];
    pa[programIndex] = ptr;

    foreach_active(i) {
        if (__is_xe_target) {
            void *uniform d = pa[i];
            for (uniform int j = 0; j < extract(count, i); j++) {
                ((int8 * uniform) d)[j] = extract(val, i);
            }
        } else {
            __memset32((uniform int8 * uniform) pa[i], extract(val, i), extract(count, i));
        }
    }
}

static inline void memset64(void *varying ptr, int8 val, int64 count) {
    void *uniform pa[programCount];
    pa[programIndex] = ptr;

    foreach_active(i) {
        if (__is_xe_target) {
            void *uniform d = pa[i];
            for (uniform int64 j = 0; j < extract(count, i); j++) {
                ((int8 * uniform) d)[j] = extract(val, i);
            }
        } else {
            __memset64((uniform int8 * uniform) pa[i], extract(val, i), extract(count, i));
        }
    }
}

///////////////////////////////////////////////////////////////////////////
// count leading/trailing zeros

__declspec(safe, cost1) static inline uniform unsigned int32 count_leading_zeros(uniform unsigned int32 v) {
    return __count_leading_zeros_uniform_i32(v);
}

__declspec(safe, cost1) static inline uniform unsigned int64 count_leading_zeros(uniform unsigned int64 v) {
    return __count_leading_zeros_uniform_i64(v);
}

__declspec(safe, cost1) static inline uniform unsigned int32 count_trailing_zeros(uniform unsigned int32 v) {
    return __count_trailing_zeros_uniform_i32(v);
}

__declspec(safe, cost1) static inline uniform unsigned int64 count_trailing_zeros(uniform unsigned int64 v) {
    return __count_trailing_zeros_uniform_i64(v);
}

__declspec(safe, cost1) static inline uniform int32 count_leading_zeros(uniform int32 v) {
    return __count_leading_zeros_uniform_i32(v);
}

__declspec(safe, cost1) static inline uniform int64 count_leading_zeros(uniform int64 v) {
    return __count_leading_zeros_uniform_i64(v);
}

__declspec(safe, cost1) static inline uniform int32 count_trailing_zeros(uniform int32 v) {
    return __count_trailing_zeros_uniform_i32(v);
}

__declspec(safe, cost1) static inline uniform int64 count_trailing_zeros(uniform int64 v) {
    return __count_trailing_zeros_uniform_i64(v);
}

__declspec(safe) static inline unsigned int32 count_leading_zeros(unsigned int32 v) {
    if (__have_conflict_detection) {
        return __count_leading_zeros_varying_i32(v);
    } else {
        unsigned int32 r;
        for (uniform int i = 0; i < programCount; ++i)
            r = insert(r, i, (uniform unsigned int32)__count_leading_zeros_uniform_i32(extract(v, i)));
        return r;
    }
}

__declspec(safe) static inline unsigned int64 count_leading_zeros(unsigned int64 v) {
    if (__have_conflict_detection) {
        return __count_leading_zeros_varying_i64(v);
    } else {
        unsigned int64 r;
        for (uniform int i = 0; i < programCount; ++i)
            r = insert(r, i, (uniform unsigned int64)__count_leading_zeros_uniform_i64(extract(v, i)));
        return r;
    }
}

__declspec(safe) static inline unsigned int32 count_trailing_zeros(unsigned int32 v) {
    // Using the vector LLVM's cttz intrinsic is more efficient than
    // counting trailing zeros in a loop. However, it is not true for ctlz -
    // a loop of uniform calculations performs better than vector LLVM's ctlz intrinsic.
    // https://github.com/llvm/llvm-project/issues/124993
    return __count_trailing_zeros_varying_i32(v);
}

__declspec(safe) static inline unsigned int64 count_trailing_zeros(unsigned int64 v) {
    // Using the vector LLVM's cttz intrinsic is more efficient than
    // counting trailing zeros in a loop. However, it is not true for ctlz -
    // a loop of uniform calculations performs better than vector LLVM's ctlz intrinsic.
    // https://github.com/llvm/llvm-project/issues/124993
    return __count_trailing_zeros_varying_i64(v);
}

__declspec(safe) static inline int32 count_leading_zeros(int32 v) {
    if (__have_conflict_detection) {
        return __count_leading_zeros_varying_i32(v);
    } else {
        int32 r;
        for (uniform int i = 0; i < programCount; ++i)
            r = insert(r, i, __count_leading_zeros_uniform_i32(extract(v, i)));
        return r;
    }
}

__declspec(safe) static inline int64 count_leading_zeros(int64 v) {
    if (__have_conflict_detection) {
        return __count_leading_zeros_varying_i64(v);
    } else {
        int64 r;
        for (uniform int i = 0; i < programCount; ++i)
            r = insert(r, i, __count_leading_zeros_uniform_i64(extract(v, i)));
        return r;
    }
}

__declspec(safe) static inline int32 count_trailing_zeros(int32 v) {
    // Using the vector LLVM's cttz intrinsic is more efficient than
    // counting trailing zeros in a loop. However, it is not true for ctlz -
    // a loop of uniform calculations performs better than vector LLVM's ctlz intrinsic.
    // https://github.com/llvm/llvm-project/issues/124993
    return __count_trailing_zeros_varying_i32(v);
}

__declspec(safe) static inline int64 count_trailing_zeros(int64 v) {
    // Using the vector LLVM's cttz intrinsic is more efficient than
    // counting trailing zeros in a loop. However, it is not true for ctlz -
    // a loop of uniform calculations performs better than vector LLVM's ctlz intrinsic.
    // https://github.com/llvm/llvm-project/issues/124993
    return __count_trailing_zeros_varying_i64(v);
}

///////////////////////////////////////////////////////////////////////////
// AOS/SOA conversion

static inline void aos_to_soa2(uniform float a[], varying float *uniform v0, varying float *uniform v1) {
    __aos_to_soa2_float((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1);
}

static inline void soa_to_aos2(float v0, float v1, uniform float a[]) { __soa_to_aos2_float(v0, v1, (opaque_ptr_t)a); }

static inline void aos_to_soa3(uniform float a[], varying float *uniform v0, varying float *uniform v1,
                               varying float *uniform v2) {
    __aos_to_soa3_float((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1, (opaque_ptr_t)v2);
}

static inline void soa_to_aos3(float v0, float v1, float v2, uniform float a[]) {
    __soa_to_aos3_float(v0, v1, v2, (opaque_ptr_t)a);
}

static inline void aos_to_soa4(uniform float a[], varying float *uniform v0, varying float *uniform v1,
                               varying float *uniform v2, varying float *uniform v3) {
    __aos_to_soa4_float((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1, (opaque_ptr_t)v2, (opaque_ptr_t)v3);
}

static inline void soa_to_aos4(float v0, float v1, float v2, float v3, uniform float a[]) {
    __soa_to_aos4_float(v0, v1, v2, v3, (opaque_ptr_t)a);
}

static inline void aos_to_soa2(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1) {
    aos_to_soa2((uniform float *uniform)a, (varying float *uniform)v0, (varying float *uniform)v1);
}

static inline void soa_to_aos2(int32 v0, int32 v1, uniform int32 a[]) {
    soa_to_aos2(floatbits(v0), floatbits(v1), (uniform float *uniform)a);
}

static inline void aos_to_soa3(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1,
                               varying int32 *uniform v2) {
    aos_to_soa3((uniform float *uniform)a, (varying float *uniform)v0, (varying float *uniform)v1,
                (varying float *uniform)v2);
}

static inline void soa_to_aos3(int32 v0, int32 v1, int32 v2, uniform int32 a[]) {
    soa_to_aos3(floatbits(v0), floatbits(v1), floatbits(v2), (uniform float *uniform)a);
}

static inline void aos_to_soa4(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1,
                               varying int32 *uniform v2, varying int32 *uniform v3) {
    aos_to_soa4((uniform float *uniform)a, (varying float *uniform)v0, (varying float *uniform)v1,
                (varying float *uniform)v2, (varying float *uniform)v3);
}

static inline void soa_to_aos4(int32 v0, int32 v1, int32 v2, int32 v3, uniform int32 a[]) {
    soa_to_aos4(floatbits(v0), floatbits(v1), floatbits(v2), floatbits(v3), (uniform float *uniform)a);
}

static inline void aos_to_soa2(uniform double a[], varying double *uniform v0, varying double *uniform v1) {
    __aos_to_soa2_double((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1);
}

static inline void soa_to_aos2(double v0, double v1, uniform double a[]) {
    __soa_to_aos2_double(v0, v1, (opaque_ptr_t)a);
}

static inline void aos_to_soa3(uniform double a[], varying double *uniform v0, varying double *uniform v1,
                               varying double *uniform v2) {
    __aos_to_soa3_double((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1, (opaque_ptr_t)v2);
}

static inline void soa_to_aos3(double v0, double v1, double v2, uniform double a[]) {
    __soa_to_aos3_double(v0, v1, v2, (opaque_ptr_t)a);
}

static inline void aos_to_soa4(uniform double a[], varying double *uniform v0, varying double *uniform v1,
                               varying double *uniform v2, varying double *uniform v3) {
    __aos_to_soa4_double((opaque_ptr_t)a, (opaque_ptr_t)v0, (opaque_ptr_t)v1, (opaque_ptr_t)v2, (opaque_ptr_t)v3);
}

static inline void soa_to_aos4(double v0, double v1, double v2, double v3, uniform double a[]) {
    __soa_to_aos4_double(v0, v1, v2, v3, (opaque_ptr_t)a);
}

static inline void aos_to_soa2(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1) {
    aos_to_soa2((uniform double *uniform)a, (varying double *uniform)v0, (varying double *uniform)v1);
}

static inline void soa_to_aos2(int64 v0, int64 v1, uniform int64 a[]) {
    soa_to_aos2(doublebits(v0), doublebits(v1), (uniform double *uniform)a);
}

static inline void aos_to_soa3(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1,
                               varying int64 *uniform v2) {
    aos_to_soa3((uniform double *uniform)a, (varying double *uniform)v0, (varying double *uniform)v1,
                (varying double *uniform)v2);
}

static inline void soa_to_aos3(int64 v0, int64 v1, int64 v2, uniform int64 a[]) {
    soa_to_aos3(doublebits(v0), doublebits(v1), doublebits(v2), (uniform double *uniform)a);
}

static inline void aos_to_soa4(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1,
                               varying int64 *uniform v2, varying int64 *uniform v3) {
    aos_to_soa4((uniform double *uniform)a, (varying double *uniform)v0, (varying double *uniform)v1,
                (varying double *uniform)v2, (varying double *uniform)v3);
}

static inline void soa_to_aos4(int64 v0, int64 v1, int64 v2, int64 v3, uniform int64 a[]) {
    soa_to_aos4(doublebits(v0), doublebits(v1), doublebits(v2), doublebits(v3), (uniform double *uniform)a);
}
///////////////////////////////////////////////////////////////////////////
// Prefetching

__declspec(safe, cost1) static inline void prefetch_l1(const void *uniform ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_1((opaque_ptr_t)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT);
    } else {
        __prefetch_read_uniform_1((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_l1(const void *uniform ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_1((opaque_ptr_t)ptr, size);
    } else {
        __prefetch_read_uniform_1((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_l2(const void *uniform ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_2((opaque_ptr_t)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT);
    } else {
        __prefetch_read_uniform_2((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_l2(const void *uniform ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_2((opaque_ptr_t)ptr, size);
    } else {
        __prefetch_read_uniform_2((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_l3(const void *uniform ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_3((opaque_ptr_t)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT);
    } else {
        __prefetch_read_uniform_3((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_l3(const void *uniform ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_3((opaque_ptr_t)ptr, size);
    } else {
        __prefetch_read_uniform_3((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_nt(const void *uniform ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_nt((opaque_ptr_t)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT);
    } else {
        __prefetch_read_uniform_nt((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetch_nt(const void *uniform ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_uniform_nt((opaque_ptr_t)ptr, size);
    } else {
        __prefetch_read_uniform_nt((opaque_ptr_t)ptr);
    }
}

__declspec(safe, cost1) static inline void prefetchw_l1(const void *uniform ptr) {
    __prefetch_write_uniform_1((opaque_ptr_t)ptr);
}

__declspec(safe, cost1) static inline void prefetchw_l2(const void *uniform ptr) {
    __prefetch_write_uniform_2((opaque_ptr_t)ptr);
}

__declspec(safe, cost1) static inline void prefetchw_l3(const void *uniform ptr) {
    __prefetch_write_uniform_3((opaque_ptr_t)ptr);
}

static inline void prefetch_l1(const void *varying ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_1((int64)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_1((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_l1(const void *varying ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_1((int64)ptr, size, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_1((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_l2(const void *varying ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_2((int64)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_2((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_l2(const void *varying ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_2((int64)ptr, size, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_2((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_l3(const void *varying ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_3((int64)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_3((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_l3(const void *varying ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_3((int64)ptr, size, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_3((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_nt(const void *varying ptr) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_nt((int64)ptr, (uniform int8)PREFETCH_DATASIZE_DEFAULT, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_nt((int64)ptr, (IntMaskType)__mask);
    }
}

static inline void prefetch_nt(const void *varying ptr, uniform int8 size) {
    if (__have_xe_prefetch) {
        __prefetch_read_sized_varying_nt((int64)ptr, size, (IntMaskType)__mask);
    } else {
        __pseudo_prefetch_read_varying_nt((int64)ptr, (IntMaskType)__mask);
    }
}

__declspec(safe, cost1) static inline void prefetchw_l1(const void *varying ptr) {
    __pseudo_prefetch_write_varying_1((int64)ptr, (IntMaskType)__mask);
}

__declspec(safe, cost1) static inline void prefetchw_l2(const void *varying ptr) {
    __pseudo_prefetch_write_varying_2((int64)ptr, (IntMaskType)__mask);
}

__declspec(safe, cost1) static inline void prefetchw_l3(const void *varying ptr) {
    __pseudo_prefetch_write_varying_3((int64)ptr, (IntMaskType)__mask);
}

///////////////////////////////////////////////////////////////////////////
// non-short-circuiting alternatives

__declspec(safe, cost1) static inline bool and (bool a, bool b) { return a && b; }

__declspec(safe, cost1) static inline uniform bool and (uniform bool a, uniform bool b) { return a && b; }

__declspec(safe, cost1) static inline bool or (bool a, bool b) { return a || b; }

__declspec(safe, cost1) static inline uniform bool or (uniform bool a, uniform bool b) { return a || b; }

__declspec(safe, cost1) static inline int8 select(bool cond, int8 t, int8 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint8 select(bool cond, uint8 t, uint8 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline int8 select(uniform bool cond, int8 t, int8 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint8 select(uniform bool cond, uint8 t, uint8 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform int8 select(uniform bool cond, uniform int8 t, uniform int8 f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline int16 select(bool cond, int16 t, int16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint16 select(bool cond, uint16 t, uint16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline int16 select(uniform bool cond, int16 t, int16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint16 select(uniform bool cond, uint16 t, uint16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform int16 select(uniform bool cond, uniform int16 t, uniform int16 f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline float16 select(bool cond, float16 t, float16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline float16 select(uniform bool cond, float16 t, float16 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform float16 select(uniform bool cond, uniform float16 t, uniform float16 f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline int32 select(bool cond, int32 t, int32 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint32 select(bool cond, uint32 t, uint32 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline int32 select(uniform bool cond, int32 t, int32 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint32 select(uniform bool cond, uint32 t, uint32 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform int32 select(uniform bool cond, uniform int32 t, uniform int32 f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline int64 select(bool cond, int64 t, int64 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint64 select(bool cond, uint64 t, uint64 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline int64 select(uniform bool cond, int64 t, int64 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uint64 select(uniform bool cond, uint64 t, uint64 f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform int64 select(uniform bool cond, uniform int64 t, uniform int64 f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline float select(bool cond, float t, float f) { return cond ? t : f; }

__declspec(safe, cost1) static inline float select(uniform bool cond, float t, float f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform float select(uniform bool cond, uniform float t, uniform float f) {
    return cond ? t : f;
}

__declspec(safe, cost1) static inline double select(bool cond, double t, double f) { return cond ? t : f; }

__declspec(safe, cost1) static inline double select(uniform bool cond, double t, double f) { return cond ? t : f; }

__declspec(safe, cost1) static inline uniform double select(uniform bool cond, uniform double t, uniform double f) {
    return cond ? t : f;
}

///////////////////////////////////////////////////////////////////////////
// Horizontal ops / reductions

__declspec(safe) static inline uniform int16 reduce_add(int8 x) { return __reduce_add_int8(__mask ? x : (int8)0); }

__declspec(safe) static inline uniform int8 reduce_min(int8 v) {
    // Set values for non-running lanes to the maximum int8 value so
    // they don't affect the result.
    return __reduce_min_int8(__mask ? v : INT8_MAX);
}

__declspec(safe) static inline uniform int8 reduce_max(int8 v) {
    // Set values for non-running lanes to the minimum int8 value so
    // they don't affect the result.
    return __reduce_max_int8(__mask ? v : INT8_MIN);
}

__declspec(safe) static inline uniform unsigned int16 reduce_add(unsigned int8 x) {
    return __reduce_add_int8(__mask ? x : (int8)0);
}

__declspec(safe) static inline uniform unsigned int8 reduce_min(unsigned int8 v) {
    // Set values for non-running lanes to the maximum uint8 value so
    // they don't affect the result.
    return __reduce_min_uint8(__mask ? v : UINT8_MAX);
}

__declspec(safe) static inline uniform unsigned int8 reduce_max(unsigned int8 v) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_max_uint8(__mask ? v : 0);
}

__declspec(safe) static inline uniform int32 reduce_add(int16 x) { return __reduce_add_int16(__mask ? x : (int16)0); }

__declspec(safe) static inline uniform int16 reduce_min(int16 v) {
    // Set values for non-running lanes to the maximum int8 value so
    // they don't affect the result.
    return __reduce_min_int16(__mask ? v : INT16_MAX);
}

__declspec(safe) static inline uniform int16 reduce_max(int16 v) {
    // Set values for non-running lanes to the minimum int8 value so
    // they don't affect the result.
    return __reduce_max_int16(__mask ? v : INT16_MIN);
}

__declspec(safe) static inline uniform unsigned int32 reduce_add(unsigned int16 x) {
    return __reduce_add_int16(__mask ? x : (int16)0);
}

__declspec(safe) static inline uniform unsigned int16 reduce_min(unsigned int16 v) {
    // Set values for non-running lanes to the maximum uint8 value so
    // they don't affect the result.
    return __reduce_min_uint16(__mask ? v : UINT16_MAX);
}

__declspec(safe) static inline uniform unsigned int16 reduce_max(unsigned int16 v) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_max_uint16(__mask ? v : 0);
}

__declspec(safe) static inline uniform float16 reduce_add(float16 x) {
    // zero the lanes where the mask is off
    return __reduce_add_half(__mask ? x : 0.0f16);
}

__declspec(safe) static inline uniform float16 reduce_min(float16 v) {
    // For the lanes where the mask is off, replace the given value with
    // infinity, so that it doesn't affect the result.
    const int16 iflt_max = 0x7c00; // infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_min_float() are calculated without a mask.
    bool test = __mask;
    uniform float16 result;
    unmasked { result = __reduce_min_half(test ? v : float16bits(iflt_max)); }
    return result;
}

__declspec(safe) static inline uniform float16 reduce_max(float16 v) {
    // For the lanes where the mask is off, replace the given value with
    // negative infinity, so that it doesn't affect the result.
    const int16 iflt_neg_max = 0xfc00; // -infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_max_half() are calculated without a mask.
    bool test = __mask;
    uniform float16 result;
    unmasked { result = __reduce_max_half(test ? v : float16bits(iflt_neg_max)); }
    return result;
}

__declspec(safe) static inline uniform float reduce_add(float x) {
    // zero the lanes where the mask is off
    return __reduce_add_float(__mask ? x : 0.);
}

__declspec(safe) static inline uniform float reduce_min(float v) {
    // For the lanes where the mask is off, replace the given value with
    // infinity, so that it doesn't affect the result.
    int iflt_max = 0x7f800000; // infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_min_float() are calculated without a mask.
    bool test = __mask;
    uniform float result;
    unmasked { result = __reduce_min_float(test ? v : floatbits(iflt_max)); }
    return result;
}

__declspec(safe) static inline uniform float reduce_max(float v) {
    // For the lanes where the mask is off, replace the given value with
    // negative infinity, so that it doesn't affect the result.
    const int iflt_neg_max = 0xff800000; // -infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_max_float() are calculated without a mask.
    bool test = __mask;
    uniform float result;
    unmasked { result = __reduce_max_float(test ? v : floatbits(iflt_neg_max)); }
    return result;
}

__declspec(safe) static inline uniform int64 reduce_add(int32 x) {
    // Zero out the values for lanes that aren't running
    return __reduce_add_int32(__mask ? x : 0);
}

__declspec(safe) static inline uniform int reduce_min(int v) {
    // Set values for non-running lanes to the maximum integer value so
    // they don't affect the result.
    return __reduce_min_int32(__mask ? v : INT32_MAX);
}

__declspec(safe) static inline uniform int reduce_max(int v) {
    // Set values for non-running lanes to the minimum integer value so
    // they don't affect the result.
    return __reduce_max_int32(__mask ? v : INT32_MIN);
}

__declspec(safe) static inline uniform unsigned int64 reduce_add(unsigned int32 x) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_add_int32(__mask ? x : 0);
}

__declspec(safe) static inline uniform unsigned int reduce_min(unsigned int v) {
    // Set values for non-running lanes to the maximum unsigned integer
    // value so they don't affect the result.
    return __reduce_min_uint32(__mask ? v : UINT32_MAX);
}

__declspec(safe) static inline uniform unsigned int reduce_max(unsigned int v) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_max_uint32(__mask ? v : 0);
}

__declspec(safe) static inline uniform double reduce_add(double x) {
    // zero the lanes where the mask is off
    return __reduce_add_double(__mask ? x : 0.);
}

__declspec(safe) static inline uniform double reduce_min(double v) {
    int64 iflt_max = 0x7ff0000000000000; // infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_min_double() are calculated without a mask.
    bool test = __mask;
    uniform double result;
    unmasked { result = __reduce_min_double(test ? v : doublebits(iflt_max)); }
    return result;
}

__declspec(safe) static inline uniform double reduce_max(double v) {
    const int64 iflt_neg_max = 0xfff0000000000000; // -infinity
    // unmasked block is needed to make sure that argument for unmasked
    // function __reduce_max_double() are calculated without a mask.
    bool test = __mask;
    uniform double result;
    unmasked { result = __reduce_max_double(test ? v : doublebits(iflt_neg_max)); }
    return result;
}

__declspec(safe) static inline uniform int64 reduce_add(int64 x) {
    // Zero out the values for lanes that aren't running
    return __reduce_add_int64(__mask ? x : 0);
}

__declspec(safe) static inline uniform int64 reduce_min(int64 v) {
    // Set values for non-running lanes to the maximum integer value so
    // they don't affect the result.
    return __reduce_min_int64(__mask ? v : INT64_MAX);
}

__declspec(safe) static inline uniform int64 reduce_max(int64 v) {
    // Set values for non-running lanes to the minimum integer value so
    // they don't affect the result.
    return __reduce_max_int64(__mask ? v : INT64_MIN);
}

__declspec(safe) static inline uniform unsigned int64 reduce_add(unsigned int64 x) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_add_int64(__mask ? x : 0);
}

__declspec(safe) static inline uniform unsigned int64 reduce_min(unsigned int64 v) {
    // Set values for non-running lanes to the maximum unsigned integer
    // value so they don't affect the result.
    return __reduce_min_uint64(__mask ? v : UINT64_MAX);
}

__declspec(safe) static inline uniform unsigned int64 reduce_max(unsigned int64 v) {
    // Set values for non-running lanes to zero so they don't affect the
    // result.
    return __reduce_max_uint64(__mask ? v : 0);
}

#define REDUCE_EQUAL(TYPE, FUNCTYPE, MASKTYPE)                                                                         \
    __declspec(safe) static inline uniform bool reduce_equal(TYPE v) {                                                 \
        uniform int8 unusedValue;                                                                                      \
        return __reduce_equal_##FUNCTYPE(v, &unusedValue, (MASKTYPE)__mask);                                           \
    }                                                                                                                  \
    __declspec(safe) static inline uniform bool reduce_equal(TYPE v, uniform TYPE *uniform value) {                    \
        return __reduce_equal_##FUNCTYPE(v, (opaque_ptr_t)value, (MASKTYPE)__mask);                                    \
    }

REDUCE_EQUAL(float16, half, IntMaskType)
REDUCE_EQUAL(int8, int8, IntMaskType)
REDUCE_EQUAL(unsigned int8, int8, UIntMaskType)
REDUCE_EQUAL(int16, int16, IntMaskType)
REDUCE_EQUAL(unsigned int16, int16, UIntMaskType)
REDUCE_EQUAL(int32, int32, IntMaskType)
REDUCE_EQUAL(unsigned int32, int32, UIntMaskType)
REDUCE_EQUAL(float, float, IntMaskType)
REDUCE_EQUAL(int64, int64, IntMaskType)
REDUCE_EQUAL(unsigned int64, int64, UIntMaskType)
REDUCE_EQUAL(double, double, IntMaskType)

static float16 exclusive_scan_add(float16 v) { return __exclusive_scan_add_half(v, __mask); }

static int8 exclusive_scan_add(int8 v) { return __exclusive_scan_add_i8(v, (IntMaskType)__mask); }

static unsigned int8 exclusive_scan_add(unsigned int8 v) {
    return __exclusive_scan_add_i8((int8)v, (IntMaskType)__mask);
}

static int16 exclusive_scan_add(int16 v) { return __exclusive_scan_add_i16(v, (IntMaskType)__mask); }

static unsigned int16 exclusive_scan_add(unsigned int16 v) {
    return __exclusive_scan_add_i16((int16)v, (IntMaskType)__mask);
}

static int32 exclusive_scan_add(int32 v) { return __exclusive_scan_add_i32(v, (IntMaskType)__mask); }

static unsigned int32 exclusive_scan_add(unsigned int32 v) {
    return __exclusive_scan_add_i32((int32)v, (IntMaskType)__mask);
}

static float exclusive_scan_add(float v) { return __exclusive_scan_add_float(v, __mask); }

static int64 exclusive_scan_add(int64 v) { return __exclusive_scan_add_i64(v, (IntMaskType)__mask); }

static unsigned int64 exclusive_scan_add(unsigned int64 v) { return __exclusive_scan_add_i64(v, (UIntMaskType)__mask); }

static double exclusive_scan_add(double v) { return __exclusive_scan_add_double(v, __mask); }

static int8 exclusive_scan_and(int8 v) { return __exclusive_scan_and_i8(v, (IntMaskType)__mask); }

static unsigned int8 exclusive_scan_and(unsigned int8 v) { return __exclusive_scan_and_i8(v, (UIntMaskType)__mask); }

static int16 exclusive_scan_and(int16 v) { return __exclusive_scan_and_i16(v, (IntMaskType)__mask); }

static unsigned int16 exclusive_scan_and(unsigned int16 v) { return __exclusive_scan_and_i16(v, (UIntMaskType)__mask); }

static int32 exclusive_scan_and(int32 v) { return __exclusive_scan_and_i32(v, (IntMaskType)__mask); }

static unsigned int32 exclusive_scan_and(unsigned int32 v) { return __exclusive_scan_and_i32(v, (UIntMaskType)__mask); }

static int64 exclusive_scan_and(int64 v) { return __exclusive_scan_and_i64(v, (IntMaskType)__mask); }

static unsigned int64 exclusive_scan_and(unsigned int64 v) { return __exclusive_scan_and_i64(v, (UIntMaskType)__mask); }

static int8 exclusive_scan_or(int8 v) { return __exclusive_scan_or_i8(v, (IntMaskType)__mask); }

static unsigned int8 exclusive_scan_or(unsigned int8 v) { return __exclusive_scan_or_i8(v, (UIntMaskType)__mask); }

static int16 exclusive_scan_or(int16 v) { return __exclusive_scan_or_i16(v, (IntMaskType)__mask); }

static unsigned int16 exclusive_scan_or(unsigned int16 v) { return __exclusive_scan_or_i16(v, (UIntMaskType)__mask); }

static int32 exclusive_scan_or(int32 v) { return __exclusive_scan_or_i32(v, (IntMaskType)__mask); }

static unsigned int32 exclusive_scan_or(unsigned int32 v) { return __exclusive_scan_or_i32(v, (UIntMaskType)__mask); }

static int64 exclusive_scan_or(int64 v) { return __exclusive_scan_or_i64(v, (IntMaskType)__mask); }

static unsigned int64 exclusive_scan_or(unsigned int64 v) { return __exclusive_scan_or_i64(v, (UIntMaskType)__mask); }

///////////////////////////////////////////////////////////////////////////
// packed load, store

/* unsigned int32 implementations. */
// unsigned int32 load.
static inline uniform int packed_load_active(uniform unsigned int a[], varying unsigned int *uniform vals) {
    return __packed_load_activei32((opaque_ptr_t)a, (opaque_ptr_t)vals, (UIntMaskType)__mask);
}

// unsigned int32 store.
static inline uniform int packed_store_active(uniform unsigned int a[], unsigned int vals) {
    return __packed_store_activei32((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

// unsigned int32 store2.
static inline uniform int packed_store_active2(uniform unsigned int a[], unsigned int vals) {
    return __packed_store_active2i32((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

/* unsigned int16 implementations. */
// unsigned int16 load.
static inline uniform int packed_load_active(uniform unsigned int16 a[], varying unsigned int16 *uniform vals) {
    return __packed_load_activei16((opaque_ptr_t)a, (opaque_ptr_t)vals, (UIntMaskType)__mask);
}

// unsigned int16 store.
static inline uniform int packed_store_active(uniform unsigned int16 a[], unsigned int16 vals) {
    return __packed_store_activei16((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

// unsigned int16 store2.
static inline uniform int packed_store_active2(uniform unsigned int16 a[], unsigned int16 vals) {
    return __packed_store_active2i16((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

/* unsigned int8 implementations. */
// unsigned int8 load.
static inline uniform int packed_load_active(uniform unsigned int8 a[], varying unsigned int8 *uniform vals) {
    return __packed_load_activei8((opaque_ptr_t)a, (opaque_ptr_t)vals, (UIntMaskType)__mask);
}

// unsigned int8 store.
static inline uniform int packed_store_active(uniform unsigned int8 a[], unsigned int8 vals) {
    return __packed_store_activei8((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

// unsigned int8 store2.
static inline uniform int packed_store_active2(uniform unsigned int8 a[], unsigned int8 vals) {
    return __packed_store_active2i8((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

/* unsigned int64 implementations. */
// unsigned int64 load.
static inline uniform int packed_load_active(uniform unsigned int64 a[], varying unsigned int64 *uniform vals) {
    return __packed_load_activei64((opaque_ptr_t)a, (opaque_ptr_t)vals, (UIntMaskType)__mask);
}

// unsigned int64 store.
static inline uniform int packed_store_active(uniform unsigned int64 a[], unsigned int64 vals) {
    return __packed_store_activei64((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

// unsigned int64 store2.
static inline uniform int packed_store_active2(uniform unsigned int64 a[], unsigned int64 vals) {
    return __packed_store_active2i64((opaque_ptr_t)a, vals, (UIntMaskType)__mask);
}

/* int32 implementations. */
// int32 load.
static inline uniform int packed_load_active(uniform int a[], varying int *uniform vals) {
    return __packed_load_activei32((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// int32 store.
static inline uniform int packed_store_active(uniform int a[], int vals) {
    return __packed_store_activei32((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int32 store2.
static inline uniform int packed_store_active2(uniform int a[], int vals) {
    return __packed_store_active2i32((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int32 store with lanes.
static inline uniform int packed_store_active(bool active, uniform int a[], int vals) {
    return __packed_store_activei32((opaque_ptr_t)a, vals, (IntMaskType)(-(int)active));
}

/* int16 implementations. */
// int16 load.
static inline uniform int packed_load_active(uniform int16 a[], varying int16 *uniform vals) {
    return __packed_load_activei16((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// int16 store.
static inline uniform int packed_store_active(uniform int16 a[], int16 vals) {
    return __packed_store_activei16((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int16 store2.
static inline uniform int packed_store_active2(uniform int16 a[], int16 vals) {
    return __packed_store_active2i16((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

/* int8 implementations. */
// int8 load.
static inline uniform int packed_load_active(uniform int8 a[], varying int8 *uniform vals) {
    return __packed_load_activei8((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// int8 store.
static inline uniform int packed_store_active(uniform int8 a[], int8 vals) {
    return __packed_store_activei8((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int8 store2.
static inline uniform int packed_store_active2(uniform int8 a[], int8 vals) {
    return __packed_store_active2i8((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

/* int64 implementations. */
// int64 load.
static inline uniform int packed_load_active(uniform int64 a[], varying int64 *uniform vals) {
    return __packed_load_activei64((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// int64 store.
static inline uniform int packed_store_active(uniform int64 a[], int64 vals) {
    return __packed_store_activei64((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int64 store2.
static inline uniform int packed_store_active2(uniform int64 a[], int64 vals) {
    return __packed_store_active2i64((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// int64 store with lanes.
static inline uniform int packed_store_active(bool active, uniform int64 a[], int64 vals) {
    return __packed_store_activei64((opaque_ptr_t)a, vals, (IntMaskType)(-(int)active));
}

/* float16 implementations. */
// float16 load.
static inline uniform int packed_load_active(uniform float16 a[], varying float16 *uniform vals) {
    return __packed_load_activef16((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// float16 store.
static inline uniform int packed_store_active(uniform float16 a[], float16 vals) {
    return __packed_store_activef16((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// float16 store2.
static inline uniform int packed_store_active2(uniform float16 a[], float16 vals) {
    return __packed_store_active2f16((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

/* float implementations. */
// float load.
static inline uniform int packed_load_active(uniform float a[], varying float *uniform vals) {
    return __packed_load_activef32((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// float store.
static inline uniform int packed_store_active(uniform float a[], float vals) {
    return __packed_store_activef32((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// float store2.
static inline uniform int packed_store_active2(uniform float a[], float vals) {
    return __packed_store_active2f32((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

/* double implementations. */
// double load.
static inline uniform int packed_load_active(uniform double a[], varying double *uniform vals) {
    return __packed_load_activef64((opaque_ptr_t)a, (opaque_ptr_t)vals, (IntMaskType)__mask);
}

// double store.
static inline uniform int packed_store_active(uniform double a[], double vals) {
    return __packed_store_activef64((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}

// double store2.
static inline uniform int packed_store_active2(uniform double a[], double vals) {
    return __packed_store_active2f64((opaque_ptr_t)a, vals, (IntMaskType)__mask);
}
///////////////////////////////////////////////////////////////////////////
// streaming store

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int8 a[], unsigned int8 vals) {
    __streaming_store_varying_i8((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int8 a[], int8 vals) {
    __streaming_store_varying_i8((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int16 a[], unsigned int16 vals) {
    __streaming_store_varying_i16((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int16 a[], int16 vals) {
    __streaming_store_varying_i16((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform float16 a[], float16 vals) {
    __streaming_store_varying_half((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int a[], unsigned int vals) {
    __streaming_store_varying_i32((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int a[], int vals) {
    __streaming_store_varying_i32((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int64 a[], unsigned int64 vals) {
    __streaming_store_varying_i64((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int64 a[], int64 vals) {
    __streaming_store_varying_i64((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform float a[], float vals) {
    __streaming_store_varying_float((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform double a[], double vals) {
    __streaming_store_varying_double((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int8 a[], uniform unsigned int8 vals) {
    __streaming_store_uniform_i8((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int8 a[], uniform int8 vals) {
    __streaming_store_uniform_i8((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int16 a[], uniform unsigned int16 vals) {
    __streaming_store_uniform_i16((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int16 a[], uniform int16 vals) {
    __streaming_store_uniform_i16((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform float16 a[], uniform float16 vals) {
    __streaming_store_uniform_half((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int a[], uniform unsigned int vals) {
    __streaming_store_uniform_i32((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int a[], uniform int vals) {
    __streaming_store_uniform_i32((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform unsigned int64 a[], uniform unsigned int64 vals) {
    __streaming_store_uniform_i64((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform int64 a[], uniform int64 vals) {
    __streaming_store_uniform_i64((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform float a[], uniform float vals) {
    __streaming_store_uniform_float((opaque_ptr_t)a, vals);
}

__declspec(safe, cost1) static inline void streaming_store(uniform double a[], uniform double vals) {
    __streaming_store_uniform_double((opaque_ptr_t)a, vals);
}

///////////////////////////////////////////////////////////////////////////
// streaming load

__declspec(safe, cost1) static inline varying unsigned int8 streaming_load(uniform unsigned int8 a[]) {
    return (unsigned int8)__streaming_load_varying_i8((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying int8 streaming_load(uniform int8 a[]) {
    return __streaming_load_varying_i8((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform unsigned int8 streaming_load_uniform(uniform unsigned int8 a[]) {
    return (unsigned int8)__streaming_load_uniform_i8((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform int8 streaming_load_uniform(uniform int8 a[]) {
    return __streaming_load_uniform_i8((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying unsigned int16 streaming_load(uniform unsigned int16 a[]) {
    return (unsigned int16)__streaming_load_varying_i16((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying int16 streaming_load(uniform int16 a[]) {
    return __streaming_load_varying_i16((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform unsigned int16 streaming_load_uniform(uniform unsigned int16 a[]) {
    return (unsigned int16)__streaming_load_uniform_i16((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform int16 streaming_load_uniform(uniform int16 a[]) {
    return __streaming_load_uniform_i16((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying float16 streaming_load(uniform float16 a[]) {
    return __streaming_load_varying_half((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform float16 streaming_load_uniform(uniform float16 a[]) {
    return __streaming_load_uniform_half((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying unsigned int streaming_load(uniform unsigned int a[]) {
    return (unsigned int)__streaming_load_varying_i32((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying int streaming_load(uniform int a[]) {
    return __streaming_load_varying_i32((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform unsigned int streaming_load_uniform(uniform unsigned int a[]) {
    return (unsigned int)__streaming_load_uniform_i32((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform int streaming_load_uniform(uniform int a[]) {
    return __streaming_load_uniform_i32((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying unsigned int64 streaming_load(uniform unsigned int64 a[]) {
    return (unsigned int64)__streaming_load_varying_i64((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying int64 streaming_load(uniform int64 a[]) {
    return __streaming_load_varying_i64((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform unsigned int64 streaming_load_uniform(uniform unsigned int64 a[]) {
    return (unsigned int64)__streaming_load_uniform_i64((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform int64 streaming_load_uniform(uniform int64 a[]) {
    return __streaming_load_uniform_i64((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying float streaming_load(uniform float a[]) {
    return __streaming_load_varying_float((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform float streaming_load_uniform(uniform float a[]) {
    return __streaming_load_uniform_float((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline varying double streaming_load(uniform double a[]) {
    return __streaming_load_varying_double((opaque_ptr_t)a);
}

__declspec(safe, cost1) static inline uniform double streaming_load_uniform(uniform double a[]) {
    return __streaming_load_uniform_double((opaque_ptr_t)a);
}

///////////////////////////////////////////////////////////////////////////
// System information

static inline uniform int num_cores() { return __num_cores(); }

__declspec(safe) static inline uniform int64 clock() { return __clock(); }

///////////////////////////////////////////////////////////////////////////
// Floating-Point Math

__declspec(safe, cost1) static inline uniform bool isnan(uniform float16 v) { return v != v; }

__declspec(safe, cost1) static inline bool isnan(float16 v) { return v != v; }

__declspec(safe, cost1) static inline uniform bool isnan(uniform float v) { return v != v; }

__declspec(safe, cost1) static inline bool isnan(float v) { return v != v; }

__declspec(safe, cost1) static inline uniform bool isnan(uniform double v) { return v != v; }

__declspec(safe, cost1) static inline bool isnan(double v) { return v != v; }

__declspec(safe, cost1) static inline uniform bool isinf(uniform float16 v) { return intbits(abs(v)) == 0x7C00; }

__declspec(safe, cost1) static inline bool isinf(float16 v) { return intbits(abs(v)) == 0x7C00; }

__declspec(safe, cost1) static inline uniform bool isinf(uniform float v) { return intbits(abs(v)) == 0x7F800000; }

__declspec(safe, cost1) static inline bool isinf(float v) { return intbits(abs(v)) == 0x7F800000; }

__declspec(safe, cost1) static inline uniform bool isinf(uniform double v) {
    return intbits(abs(v)) == 0x7FF0000000000000;
}

__declspec(safe, cost1) static inline bool isinf(double v) { return intbits(abs(v)) == 0x7FF0000000000000; }

__declspec(safe, cost1) static inline uniform bool isfinite(uniform float16 v) {
    return (intbits(v) & 0x7C00) != 0x7C00;
}

__declspec(safe, cost1) static inline bool isfinite(float16 v) { return (intbits(v) & 0x7C00) != 0x7C00; }

__declspec(safe, cost1) static inline uniform bool isfinite(uniform float v) {
    return (intbits(v) & 0x7F800000) != 0x7F800000;
}

__declspec(safe, cost1) static inline bool isfinite(float v) { return (intbits(v) & 0x7F800000) != 0x7F800000; }

__declspec(safe, cost1) static inline uniform bool isfinite(uniform double v) {
    return (intbits(v) & 0x7FF0000000000000) != 0x7FF0000000000000;
}

__declspec(safe, cost1) static inline bool isfinite(double v) {
    return (intbits(v) & 0x7FF0000000000000) != 0x7FF0000000000000;
}

__declspec(safe, cost1) static inline int8 abs(int8 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline uniform int8 abs(uniform int8 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline int16 abs(int16 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline uniform int16 abs(uniform int16 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline int abs(int a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline uniform int abs(uniform int a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline int64 abs(int64 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline uniform int64 abs(uniform int64 a) { return a > 0 ? a : -a; }

__declspec(safe, cost1) static inline float16 abs(float16 a) {
    // Floating-point hack: zeroing the high bit clears the sign
    unsigned int16 i = intbits(a);
    i &= 0x7fff;
    return float16bits(i);
}

__declspec(safe, cost1) static inline uniform float16 abs(uniform float16 a) {
    uniform unsigned int16 i = intbits(a);
    i &= 0x7fff;
    return float16bits(i);
}

__declspec(safe, cost1) static inline float abs(float a) {
    // Floating-point hack: zeroing the high bit clears the sign
    unsigned int i = intbits(a);
    i &= 0x7fffffff;
    return floatbits(i);
}

__declspec(safe, cost1) static inline uniform float abs(uniform float a) {
    uniform unsigned int i = intbits(a);
    i &= 0x7fffffff;
    return floatbits(i);
}

__declspec(safe, cost1) static inline double abs(double a) {
    // zeroing the high bit clears the sign
    unsigned int64 i = intbits(a);
    i &= 0x7fffffffffffffff;
    return doublebits(i);
}

__declspec(safe, cost1) static inline uniform double abs(uniform double a) {
    uniform unsigned int64 i = intbits(a);
    i &= 0x7fffffffffffffff;
    return doublebits(i);
}

__declspec(safe, cost1) static inline unsigned int16 signbits(float16 x) {
    unsigned int16 i = intbits(x);
    return (i & 0x8000);
}

__declspec(safe, cost1) static inline uniform unsigned int16 signbits(uniform float16 x) {
    uniform unsigned int16 i = intbits(x);
    return (i & 0x8000);
}

__declspec(safe, cost1) static inline unsigned int signbits(float x) {
    unsigned int i = intbits(x);
    return (i & 0x80000000);
}

__declspec(safe, cost1) static inline uniform unsigned int signbits(uniform float x) {
    uniform unsigned int i = intbits(x);
    return (i & 0x80000000);
}

__declspec(safe, cost1) static inline unsigned int64 signbits(double x) {
    unsigned int64 i = intbits(x);
    return (i & 0x8000000000000000);
}

__declspec(safe, cost1) static inline uniform unsigned int64 signbits(uniform double x) {
    uniform unsigned int64 i = intbits(x);
    return (i & 0x8000000000000000);
}

__declspec(safe, cost2) static inline float16 round(float16 x) { return __round_varying_half(x); }

__declspec(safe, cost2) static inline uniform float16 round(uniform float16 x) { return __round_uniform_half(x); }

__declspec(safe, cost2) static inline float round(float x) { return __round_varying_float(x); }

__declspec(safe, cost2) static inline uniform float round(uniform float x) { return __round_uniform_float(x); }

__declspec(safe, cost2) static inline double round(double x) { return __round_varying_double(x); }

__declspec(safe, cost2) static inline uniform double round(uniform double x) { return __round_uniform_double(x); }

__declspec(safe, cost2) static inline float16 floor(float16 x) { return __floor_varying_half(x); }

__declspec(safe, cost2) static inline uniform float16 floor(uniform float16 x) { return __floor_uniform_half(x); }

__declspec(safe, cost2) static inline float floor(float x) { return __floor_varying_float(x); }

__declspec(safe, cost2) static inline uniform float floor(uniform float x) { return __floor_uniform_float(x); }

__declspec(safe, cost2) static inline double floor(double x) { return __floor_varying_double(x); }

__declspec(safe, cost2) static inline uniform double floor(uniform double x) { return __floor_uniform_double(x); }

__declspec(safe, cost2) static inline float16 ceil(float16 x) { return __ceil_varying_half(x); }

__declspec(safe, cost2) static inline uniform float16 ceil(uniform float16 x) { return __ceil_uniform_half(x); }

__declspec(safe, cost2) static inline float ceil(float x) { return __ceil_varying_float(x); }

__declspec(safe, cost2) static inline uniform float ceil(uniform float x) { return __ceil_uniform_float(x); }

__declspec(safe, cost2) static inline double ceil(double x) { return __ceil_varying_double(x); }

__declspec(safe, cost2) static inline uniform double ceil(uniform double x) { return __ceil_uniform_double(x); }
///////////////////////////
__declspec(safe, cost2) static inline float16 trunc(float16 x) { return __trunc_varying_half(x); }

__declspec(safe, cost2) static inline uniform float16 trunc(uniform float16 x) { return __trunc_uniform_half(x); }

__declspec(safe, cost2) static inline float trunc(float x) { return __trunc_varying_float(x); }

__declspec(safe, cost2) static inline uniform float trunc(uniform float x) { return __trunc_uniform_float(x); }

__declspec(safe, cost2) static inline double trunc(double x) { return __trunc_varying_double(x); }

__declspec(safe, cost2) static inline uniform double trunc(uniform double x) { return __trunc_uniform_double(x); }

__declspec(safe) static inline float rcp(float v) { return __rcp_varying_float(v); }

__declspec(safe) static inline uniform float rcp(uniform float v) { return __rcp_uniform_float(v); }

__declspec(safe) static inline float rcp_fast(float v) { return __rcp_fast_varying_float(v); }

__declspec(safe) static inline uniform float rcp_fast(uniform float v) { return __rcp_fast_uniform_float(v); }

#define RCPD(QUAL)                                                                                                     \
    __declspec(safe) static inline QUAL double __rcp_iterate_##QUAL##_double(QUAL double v, QUAL double iv) {          \
        iv = iv * (2.0d - v * iv);                                                                                     \
        iv = iv * (2.0d - v * iv);                                                                                     \
        return iv;                                                                                                     \
    }                                                                                                                  \
    __declspec(safe) static inline QUAL double __rcp_safe_##QUAL##_double(QUAL double x) {                             \
        if (x <= 1.0e+33d && x >= 1.0e-33d)                                                                            \
            return __rcp_iterate_##QUAL##_double(x, rcp((QUAL float)x));                                               \
        QUAL int64 ex = intbits(x) & 0x7fe0000000000000;                                                               \
        QUAL double exp = doublebits(0x7fd0000000000000 + ~ex);                                                        \
        QUAL double y = rcp((QUAL float)(x * exp));                                                                    \
        return __rcp_iterate_##QUAL##_double(x, y * exp);                                                              \
    }

RCPD(varying)
__declspec(safe) static inline double rcp(double v) {
    if (__have_native_rcpd)
        return __rcp_varying_double(v);
    else
        return __rcp_safe_varying_double(v);
}

RCPD(uniform)
__declspec(safe) static inline uniform double rcp(uniform double v) {
    if (__have_native_rcpd)
        return __rcp_uniform_double(v);
    else
        return __rcp_safe_uniform_double(v);
}

__declspec(safe) static inline double rcp_fast(double v) {
    if (__have_native_rcpd) {
        return __rcp_fast_varying_double(v);
    } else {
        return __rcp_safe_varying_double(v);
    }
}

__declspec(safe) static inline uniform double rcp_fast(uniform double v) {
    if (__have_native_rcpd) {
        return __rcp_fast_uniform_double(v);
    } else {
        return __rcp_safe_uniform_double(v);
    }
}

__declspec(safe) static inline float16 rcp(float16 v) {
    if (__have_native_half_full_support) {
        return __rcp_varying_half(v);
    } else {
        return (float16)(rcp((float)v));
    }
}

__declspec(safe) static inline uniform float16 rcp(uniform float16 v) {
    if (__have_native_half_full_support) {
        return __rcp_uniform_half(v);
    } else {
        return (uniform float16)(rcp((uniform float)v));
    }
}

__declspec(safe, cost2) static inline float16 fmod(float16 x, float16 y) {
    // Handle special cases: NaN inputs or y == 0.0f16
    if (isnan(x) || isnan(y) || y == 0.0f16) {
        // Return NaN with the sign of x
        uniform int16 nan_bits = 0x7E00; // Quiet NaN for float16
        if (signbits(x)) {
            nan_bits |= 0x8000; // Set sign bit if x is negative
        }
        return float16bits(nan_bits);
    }

    // Handle cases where y is infinite
    int16 y_abs_bits = intbits(y) & (int16)0x7FFF; // Absolute value of y bits
    if (y_abs_bits == 0x7C00) {                    // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    int16 x_abs_bits = intbits(x) & (int16)0x7FFF; // Absolute value of x bits
    if (x_abs_bits == 0x7C00) {                    // x is infinite
        // Return NaN with the sign of x
        uniform int16 nan_bits = 0x7E00; // Quiet NaN for float16
        if (signbits(x)) {
            nan_bits |= 0x8000; // Set sign bit if x is negative
        }
        return float16bits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0f16) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

__declspec(safe, cost2) static inline uniform float16 fmod(uniform float16 x, uniform float16 y) {
    // Handle special cases: NaN inputs or y == 0.0f16
    if (isnan(x) || isnan(y) || y == 0.0f16) {
        // Return NaN with the sign of x
        uniform int16 nan_bits = 0x7E00; // Quiet NaN for float16
        if (signbits(x)) {
            nan_bits |= 0x8000; // Set sign bit if x is negative
        }
        return float16bits(nan_bits);
    }

    // Handle cases where y is infinite
    uniform int16 y_abs_bits = intbits(y) & (uniform int16)0x7FFF; // Absolute value of y bits
    if (y_abs_bits == 0x7C00) {                                    // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    uniform int16 x_abs_bits = intbits(x) & (uniform int16)0x7FFF; // Absolute value of x bits
    if (x_abs_bits == 0x7C00) {                                    // x is infinite
        // Return NaN with the sign of x
        uniform int16 nan_bits = 0x7E00; // Quiet NaN for float16
        if (signbits(x)) {
            nan_bits |= 0x8000; // Set sign bit if x is negative
        }
        return float16bits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0f16) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

__declspec(safe, cost2) static inline float fmod(float x, float y) {
    // Handle special cases: NaN inputs or y == 0.0f
    if (isnan(x) || isnan(y) || y == 0.0f) {
        // Return NaN with the sign of x
        uniform int nan_bits = 0x7FC00000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x80000000; // Set sign bit if x is negative
        }
        return floatbits(nan_bits);
    }

    // Handle cases where y is infinite
    int y_abs_bits = intbits(y) & (int)0x7FFFFFFF; // Absolute value of y bits
    if (y_abs_bits == 0x7F800000) {                // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    int x_abs_bits = intbits(x) & (int)0x7FFFFFFF; // Absolute value of x bits
    if (x_abs_bits == 0x7F800000) {                // x is infinite
        // Return NaN with the sign of x
        uniform int nan_bits = 0x7FC00000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x80000000; // Set sign bit if x is negative
        }
        return floatbits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0f) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

__declspec(safe, cost2) static inline uniform float fmod(uniform float x, uniform float y) {
    // Handle special cases: NaN inputs or y == 0.0f
    if (isnan(x) || isnan(y) || y == 0.0f) {
        // Return NaN with the sign of x
        uniform int nan_bits = 0x7FC00000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x80000000; // Set sign bit if x is negative
        }
        return floatbits(nan_bits);
    }

    // Handle cases where y is infinite
    uniform int y_abs_bits = intbits(y) & (uniform int)0x7FFFFFFF; // Absolute value of y bits
    if (y_abs_bits == 0x7F800000) {                                // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    uniform int x_abs_bits = intbits(x) & (uniform int)0x7FFFFFFF; // Absolute value of x bits
    if (x_abs_bits == 0x7F800000) {                                // x is infinite
        // Return NaN with the sign of x
        uniform int nan_bits = 0x7FC00000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x80000000; // Set sign bit if x is negative
        }
        return floatbits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0f) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

__declspec(safe, cost2) static inline double fmod(double x, double y) {
    // Handle special cases: NaN inputs or y == 0.0
    if (isnan(x) || isnan(y) || y == 0.0d) {
        // Return NaN with the sign of x
        uniform int64 nan_bits = 0x7FF8000000000000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x8000000000000000; // Set sign bit if x is negative
        }
        return doublebits(nan_bits);
    }

    // Handle cases where y is infinite
    int64 y_abs_bits = intbits(y) & (int64)0x7FFFFFFFFFFFFFFF; // Absolute value of y bits
    if (y_abs_bits == 0x7FF0000000000000) {                    // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    int64 x_abs_bits = intbits(x) & (int64)0x7FFFFFFFFFFFFFFF; // Absolute value of x bits
    if (x_abs_bits == 0x7FF0000000000000) {                    // x is infinite
        // Return NaN with the sign of x
        uniform int64 nan_bits = 0x7FF8000000000000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x8000000000000000; // Set sign bit if x is negative
        }
        return doublebits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0d) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

__declspec(safe, cost2) static inline uniform double fmod(uniform double x, uniform double y) {
    // Handle special cases: NaN inputs or y == 0.0
    if (isnan(x) || isnan(y) || y == 0.0d) {
        // Return NaN with the sign of x
        uniform int64 nan_bits = 0x7FF8000000000000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x8000000000000000; // Set sign bit if x is negative
        }
        return doublebits(nan_bits);
    }

    // Handle cases where y is infinite
    uniform int64 y_abs_bits = intbits(y) & (uniform int64)0x7FFFFFFFFFFFFFFF; // Absolute value of y bits
    if (y_abs_bits == 0x7FF0000000000000) {                                    // y is infinite
        return x;
    }

    // Handle cases where x is infinite
    uniform int64 x_abs_bits = intbits(x) & (uniform int64)0x7FFFFFFFFFFFFFFF; // Absolute value of x bits
    if (x_abs_bits == 0x7FF0000000000000) {                                    // x is infinite
        // Return NaN with the sign of x
        uniform int64 nan_bits = 0x7FF8000000000000; // Quiet NaN
        if (signbits(x)) {
            nan_bits |= 0x8000000000000000; // Set sign bit if x is negative
        }
        return doublebits(nan_bits);
    }

    // Handle cases where x is zero
    if (x == 0.0d) {
        return x;
    }

    // Compute the floating-point remainder
    return x - trunc(x * rcp(y)) * y;
}

///////////////////////////////////////////////////////////////////////////
// min/max

// float16

__declspec(safe, cost1) static inline float16 min(float16 a, float16 b) { return __min_varying_half(a, b); }

__declspec(safe, cost1) static inline uniform float16 min(uniform float16 a, uniform float16 b) {
    return __min_uniform_half(a, b);
}

__declspec(safe, cost1) static inline float16 max(float16 a, float16 b) { return __max_varying_half(a, b); }

__declspec(safe, cost1) static inline uniform float16 max(uniform float16 a, uniform float16 b) {
    return __max_uniform_half(a, b);
}

// float

__declspec(safe, cost1) static inline float min(float a, float b) { return __min_varying_float(a, b); }

__declspec(safe, cost1) static inline uniform float min(uniform float a, uniform float b) {
    return __min_uniform_float(a, b);
}

__declspec(safe, cost1) static inline float max(float a, float b) { return __max_varying_float(a, b); }

__declspec(safe, cost1) static inline uniform float max(uniform float a, uniform float b) {
    return __max_uniform_float(a, b);
}

// double

__declspec(safe) static inline double min(double a, double b) { return __min_varying_double(a, b); }

__declspec(safe) static inline uniform double min(uniform double a, uniform double b) {
    return __min_uniform_double(a, b);
}

__declspec(safe) static inline double max(double a, double b) { return __max_varying_double(a, b); }

__declspec(safe) static inline uniform double max(uniform double a, uniform double b) {
    return __max_uniform_double(a, b);
}

// int8

__declspec(safe, cost1) static inline uniform unsigned int8 min(uniform unsigned int8 a, uniform unsigned int8 b) {
    return __min_uniform_uint8(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int8 max(uniform unsigned int8 a, uniform unsigned int8 b) {
    return __max_uniform_uint8(a, b);
}

__declspec(safe, cost1) static inline uniform int8 min(uniform int8 a, uniform int8 b) {
    return __min_uniform_int8(a, b);
}

__declspec(safe, cost1) static inline uniform int8 max(uniform int8 a, uniform int8 b) {
    return __max_uniform_int8(a, b);
}

__declspec(safe, cost1) static inline unsigned int8 min(unsigned int8 a, unsigned int8 b) {
    return __min_varying_uint8(a, b);
}

__declspec(safe, cost1) static inline unsigned int8 max(unsigned int8 a, unsigned int8 b) {
    return __max_varying_uint8(a, b);
}

__declspec(safe, cost1) static inline int8 min(int8 a, int8 b) { return __min_varying_int8(a, b); }

__declspec(safe, cost1) static inline int8 max(int8 a, int8 b) { return __max_varying_int8(a, b); }

// int16

__declspec(safe, cost1) static inline uniform unsigned int16 min(uniform unsigned int16 a, uniform unsigned int16 b) {
    return __min_uniform_uint16(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int16 max(uniform unsigned int16 a, uniform unsigned int16 b) {
    return __max_uniform_uint16(a, b);
}

__declspec(safe, cost1) static inline uniform int16 min(uniform int16 a, uniform int16 b) {
    return __min_uniform_int16(a, b);
}

__declspec(safe, cost1) static inline uniform int16 max(uniform int16 a, uniform int16 b) {
    return __max_uniform_int16(a, b);
}

__declspec(safe, cost1) static inline unsigned int16 min(unsigned int16 a, unsigned int16 b) {
    return __min_varying_uint16(a, b);
}

__declspec(safe, cost1) static inline unsigned int16 max(unsigned int16 a, unsigned int16 b) {
    return __max_varying_uint16(a, b);
}

__declspec(safe, cost1) static inline int16 min(int16 a, int16 b) { return __min_varying_int16(a, b); }

__declspec(safe, cost1) static inline int16 max(int16 a, int16 b) { return __max_varying_int16(a, b); }

// int32

__declspec(safe, cost1) static inline unsigned int min(unsigned int a, unsigned int b) {
    return __min_varying_uint32(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int min(uniform unsigned int a, uniform unsigned int b) {
    return __min_uniform_uint32(a, b);
}

__declspec(safe, cost1) static inline unsigned int max(unsigned int a, unsigned int b) {
    return __max_varying_uint32(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int max(uniform unsigned int a, uniform unsigned int b) {
    return __max_uniform_uint32(a, b);
}

__declspec(safe, cost1) static inline int min(int a, int b) { return __min_varying_int32(a, b); }

__declspec(safe, cost1) static inline uniform int min(uniform int a, uniform int b) {
    return __min_uniform_int32(a, b);
}

__declspec(safe, cost1) static inline int max(int a, int b) { return __max_varying_int32(a, b); }

__declspec(safe, cost1) static inline uniform int max(uniform int a, uniform int b) {
    return __max_uniform_int32(a, b);
}

// int64

__declspec(safe, cost1) static inline unsigned int64 min(unsigned int64 a, unsigned int64 b) {
    return __min_varying_uint64(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int64 min(uniform unsigned int64 a, uniform unsigned int64 b) {
    return __min_uniform_uint64(a, b);
}

__declspec(safe, cost1) static inline unsigned int64 max(unsigned int64 a, unsigned int64 b) {
    return __max_varying_uint64(a, b);
}

__declspec(safe, cost1) static inline uniform unsigned int64 max(uniform unsigned int64 a, uniform unsigned int64 b) {
    return __max_uniform_uint64(a, b);
}

__declspec(safe, cost1) static inline int64 min(int64 a, int64 b) { return __min_varying_int64(a, b); }

__declspec(safe, cost1) static inline uniform int64 min(uniform int64 a, uniform int64 b) {
    return __min_uniform_int64(a, b);
}

__declspec(safe, cost1) static inline int64 max(int64 a, int64 b) { return __max_varying_int64(a, b); }

__declspec(safe, cost1) static inline uniform int64 max(uniform int64 a, uniform int64 b) {
    return __max_uniform_int64(a, b);
}

///////////////////////////////////////////////////////////////////////////
// clamps

// float16

__declspec(safe, cost2) static inline float16 clamp(float16 v, float16 low, float16 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline uniform float16 clamp(uniform float16 v, uniform float16 low,
                                                            uniform float16 high) {
    return min(max(v, low), high);
}

// float

__declspec(safe, cost2) static inline float clamp(float v, float low, float high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform float clamp(uniform float v, uniform float low, uniform float high) {
    return min(max(v, low), high);
}

// double

__declspec(safe, cost2) static inline double clamp(double v, double low, double high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform double clamp(uniform double v, uniform double low, uniform double high) {
    return min(max(v, low), high);
}

// int8

__declspec(safe, cost2) static inline unsigned int8 clamp(unsigned int8 v, unsigned int8 low, unsigned int8 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline uniform unsigned int8 clamp(uniform unsigned int8 v, uniform unsigned int8 low,
                                                                  uniform unsigned int8 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline int8 clamp(int8 v, int8 low, int8 high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform int8 clamp(uniform int8 v, uniform int8 low, uniform int8 high) {
    return min(max(v, low), high);
}

// int16

__declspec(safe, cost2) static inline unsigned int16 clamp(unsigned int16 v, unsigned int16 low, unsigned int16 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline uniform unsigned int16 clamp(uniform unsigned int16 v, uniform unsigned int16 low,
                                                                   uniform unsigned int16 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline int16 clamp(int16 v, int16 low, int16 high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform int16 clamp(uniform int16 v, uniform int16 low, uniform int16 high) {
    return min(max(v, low), high);
}

// int32

__declspec(safe, cost2) static inline unsigned int clamp(unsigned int v, unsigned int low, unsigned int high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline uniform unsigned int clamp(uniform unsigned int v, uniform unsigned int low,
                                                                 uniform unsigned int high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline int clamp(int v, int low, int high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform int clamp(uniform int v, uniform int low, uniform int high) {
    return min(max(v, low), high);
}

// int64

__declspec(safe, cost2) static inline unsigned int64 clamp(unsigned int64 v, unsigned int64 low, unsigned int64 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline uniform unsigned int64 clamp(uniform unsigned int64 v, uniform unsigned int64 low,
                                                                   uniform unsigned int64 high) {
    return min(max(v, low), high);
}

__declspec(safe, cost2) static inline int64 clamp(int64 v, int64 low, int64 high) { return min(max(v, low), high); }

__declspec(safe, cost2) static inline uniform int64 clamp(uniform int64 v, uniform int64 low, uniform int64 high) {
    return min(max(v, low), high);
}

///////////////////////////////////////////////////////////////////////////
// Global atomics and memory barriers

static inline void memory_barrier() { __memory_barrier(); }

#define DEFINE_ATOMIC_OP(TA, TB, OPA, OPB, MASKTYPE, TC)                                                               \
    static inline TA atomic_##OPA##_global(uniform TA *uniform ptr, TA value) {                                        \
        TA ret = __atomic_##OPB##_##TB##_global((opaque_ptr_t)ptr, value, (MASKTYPE)__mask);                           \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline uniform TA atomic_##OPA##_global(uniform TA *uniform ptr, uniform TA value) {                        \
        uniform TA ret = __atomic_##OPB##_uniform_##TB##_global((opaque_ptr_t)ptr, value);                             \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TA atomic_##OPA##_global(uniform TA *varying ptr, TA value) {                                        \
        uniform TA *uniform ptrArray[programCount];                                                                    \
        ptrArray[programIndex] = ptr;                                                                                  \
        TA ret;                                                                                                        \
        foreach_active(i) {                                                                                            \
            uniform int8 *uniform p = (opaque_ptr_t)ptrArray[i];                                                       \
            uniform TA v = extract(value, i);                                                                          \
            uniform TA r = __atomic_##OPB##_uniform_##TB##_global(p, v);                                               \
            ret = insert(ret, i, r);                                                                                   \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

#define DEFINE_ATOMIC_SWAP(TA, TB, MASKTYPE, TC)                                                                       \
    static inline TA atomic_swap_global(uniform TA *uniform ptr, TA value) {                                           \
        uniform int i = 0;                                                                                             \
        TA ret[programCount];                                                                                          \
        TA memVal;                                                                                                     \
        uniform int lastSwap;                                                                                          \
        uniform unsigned int64 mask = lanemask();                                                                      \
        /* First, have the first running program instance (if any) perform                                             \
           the swap with memory with its value of "value"; record the                                                  \
           value returned. */                                                                                          \
        for (; i < programCount; ++i) {                                                                                \
            if ((mask & (1ull << i)) == 0)                                                                             \
                continue;                                                                                              \
            memVal = __atomic_swap_uniform_##TB##_global((opaque_ptr_t)ptr, extract(value, i));                        \
            lastSwap = i;                                                                                              \
            break;                                                                                                     \
        }                                                                                                              \
        /* Now, for all of the remaining running program instances, set the                                            \
           return value of the last instance that did a swap with this                                                 \
           instance's value of "value"; this gives the same effect as if the                                           \
           current instance had executed a hardware atomic swap right before                                           \
           the last one that did a swap. */                                                                            \
        for (; i < programCount; ++i) {                                                                                \
            if ((mask & (1ull << i)) == 0)                                                                             \
                continue;                                                                                              \
            ret[lastSwap] = extract(value, i);                                                                         \
            lastSwap = i;                                                                                              \
        }                                                                                                              \
        /* And the last instance that wanted to swap gets the value we                                                 \
           originally got back from memory... */                                                                       \
        ret[lastSwap] = memVal;                                                                                        \
        return ret[programIndex];                                                                                      \
    }                                                                                                                  \
    static inline uniform TA atomic_swap_global(uniform TA *uniform ptr, uniform TA value) {                           \
        uniform TA ret = __atomic_swap_uniform_##TB##_global((opaque_ptr_t)ptr, value);                                \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TA atomic_swap_global(uniform TA *varying ptr, TA value) {                                           \
        uniform TA *uniform ptrArray[programCount];                                                                    \
        ptrArray[programIndex] = ptr;                                                                                  \
        TA ret;                                                                                                        \
        foreach_active(i) {                                                                                            \
            uniform int8 *uniform p = (opaque_ptr_t)ptrArray[i];                                                       \
            uniform TA v = extract(value, i);                                                                          \
            uniform TA r = __atomic_swap_uniform_##TB##_global(p, v);                                                  \
            ret = insert(ret, i, r);                                                                                   \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

#define DEFINE_ATOMIC_MINMAX_OP(TA, TB, OPA, OPB, MASKTYPE, TC)                                                        \
    static inline TA atomic_##OPA##_global(uniform TA *uniform ptr, TA value) {                                        \
        uniform TA oneval = reduce_##OPA(value);                                                                       \
        TA ret;                                                                                                        \
        if (lanemask() != 0)                                                                                           \
            ret = __atomic_##OPB##_uniform_##TB##_global((opaque_ptr_t)ptr, oneval);                                   \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline uniform TA atomic_##OPA##_global(uniform TA *uniform ptr, uniform TA value) {                        \
        uniform TA ret = __atomic_##OPB##_uniform_##TB##_global((opaque_ptr_t)ptr, value);                             \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TA atomic_##OPA##_global(uniform TA *varying ptr, TA value) {                                        \
        uniform TA *uniform ptrArray[programCount];                                                                    \
        ptrArray[programIndex] = ptr;                                                                                  \
        TA ret;                                                                                                        \
        foreach_active(i) {                                                                                            \
            uniform int8 *uniform p = (opaque_ptr_t)ptrArray[i];                                                       \
            uniform TA v = extract(value, i);                                                                          \
            uniform TA r = __atomic_##OPB##_uniform_##TB##_global(p, v);                                               \
            ret = insert(ret, i, r);                                                                                   \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

DEFINE_ATOMIC_OP(int32, int32, add, add, IntMaskType, int64)
DEFINE_ATOMIC_OP(int32, int32, subtract, sub, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(int32, int32, min, min, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(int32, int32, max, max, IntMaskType, int64)
DEFINE_ATOMIC_OP(int32, int32, and, and, IntMaskType, int64)
DEFINE_ATOMIC_OP(int32, int32, or, or, IntMaskType, int64)
DEFINE_ATOMIC_OP(int32, int32, xor, xor, IntMaskType, int64)
DEFINE_ATOMIC_SWAP(int32, int32, IntMaskType, int64)

// For everything but atomic min and max, we can use the same
// implementations for unsigned as for signed.
DEFINE_ATOMIC_OP(unsigned int32, int32, add, add, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int32, int32, subtract, sub, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_MINMAX_OP(unsigned int32, uint32, min, umin, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_MINMAX_OP(unsigned int32, uint32, max, umax, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int32, int32, and, and, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int32, int32, or, or, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int32, int32, xor, xor, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_SWAP(unsigned int32, int32, UIntMaskType, unsigned int64)

DEFINE_ATOMIC_OP(float, float, add, fadd, IntMaskType, int64)
DEFINE_ATOMIC_OP(float, float, subtract, fsub, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(float, float, min, fmin, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(float, float, max, fmax, IntMaskType, int64)
DEFINE_ATOMIC_SWAP(float, float, IntMaskType, int64)

DEFINE_ATOMIC_OP(int64, int64, add, add, IntMaskType, int64)
DEFINE_ATOMIC_OP(int64, int64, subtract, sub, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(int64, int64, min, min, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(int64, int64, max, max, IntMaskType, int64)
DEFINE_ATOMIC_OP(int64, int64, and, and, IntMaskType, int64)
DEFINE_ATOMIC_OP(int64, int64, or, or, IntMaskType, int64)
DEFINE_ATOMIC_OP(int64, int64, xor, xor, IntMaskType, int64)
DEFINE_ATOMIC_SWAP(int64, int64, IntMaskType, int64)

// For everything but atomic min and max, we can use the same
// implementations for unsigned as for signed.
DEFINE_ATOMIC_OP(unsigned int64, int64, add, add, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int64, int64, subtract, sub, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_MINMAX_OP(unsigned int64, uint64, min, umin, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_MINMAX_OP(unsigned int64, uint64, max, umax, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int64, int64, and, and, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int64, int64, or, or, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_OP(unsigned int64, int64, xor, xor, UIntMaskType, unsigned int64)
DEFINE_ATOMIC_SWAP(unsigned int64, int64, UIntMaskType, unsigned int64)

DEFINE_ATOMIC_OP(double, double, add, fadd, IntMaskType, int64)
DEFINE_ATOMIC_OP(double, double, subtract, fsub, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(double, double, min, fmin, IntMaskType, int64)
DEFINE_ATOMIC_MINMAX_OP(double, double, max, fmax, IntMaskType, int64)
DEFINE_ATOMIC_SWAP(double, double, IntMaskType, int64)

#undef DEFINE_ATOMIC_OP
#undef DEFINE_ATOMIC_MINMAX_OP
#undef DEFINE_ATOMIC_SWAP

#define ATOMIC_DECL_CMPXCHG(TA, TB, MASKTYPE, TC)                                                                      \
    static inline uniform TA atomic_compare_exchange_global(uniform TA *uniform ptr, uniform TA oldval,                \
                                                            uniform TA newval) {                                       \
        uniform TA ret = __atomic_compare_exchange_uniform_##TB##_global((opaque_ptr_t)ptr, oldval, newval);           \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TA atomic_compare_exchange_global(uniform TA *uniform ptr, TA oldval, TA newval) {                   \
        TA ret = __atomic_compare_exchange_##TB##_global((opaque_ptr_t)ptr, oldval, newval, (MASKTYPE)__mask);         \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TA atomic_compare_exchange_global(uniform TA *varying ptr, TA oldval, TA newval) {                   \
        uniform TA *uniform ptrArray[programCount];                                                                    \
        ptrArray[programIndex] = ptr;                                                                                  \
        TA ret;                                                                                                        \
        foreach_active(i) {                                                                                            \
            uniform TA r = __atomic_compare_exchange_uniform_##TB##_global((opaque_ptr_t)ptrArray[i],                  \
                                                                           extract(oldval, i), extract(newval, i));    \
            ret = insert(ret, i, r);                                                                                   \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

ATOMIC_DECL_CMPXCHG(int32, int32, IntMaskType, int64)
ATOMIC_DECL_CMPXCHG(unsigned int32, int32, UIntMaskType, unsigned int64)
ATOMIC_DECL_CMPXCHG(float, float, IntMaskType, int64)
ATOMIC_DECL_CMPXCHG(int64, int64, IntMaskType, int64)
ATOMIC_DECL_CMPXCHG(unsigned int64, int64, UIntMaskType, unsigned int64)
ATOMIC_DECL_CMPXCHG(double, double, IntMaskType, int64)

#undef ATOMIC_DECL_CMPXCHG

// void * variants of swap and compare exchange

static inline void *atomic_swap_global(void **uniform ptr, void *value) {
    return (void *)atomic_swap_global((uniform intptr_t * uniform) ptr, (intptr_t)value);
}

static inline void *uniform atomic_swap_global(void **uniform ptr, void *uniform value) {
    return (void *uniform)atomic_swap_global((uniform intptr_t * uniform) ptr, (uniform intptr_t)value);
}

static inline void *atomic_swap_global(void **ptr, void *value) {
    return (void *)atomic_swap_global((intptr_t *)ptr, (intptr_t)value);
}

static inline void *atomic_compare_exchange_global(void **uniform ptr, void *oldval, void *newval) {
    return (void *)atomic_compare_exchange_global((uniform intptr_t * uniform) ptr, (intptr_t)oldval, (intptr_t)newval);
}

static inline void *uniform atomic_compare_exchange_global(void **uniform ptr, void *uniform oldval,
                                                           void *uniform newval) {
    return (void *uniform)atomic_compare_exchange_global((uniform intptr_t * uniform) ptr, (uniform intptr_t)oldval,
                                                         (uniform intptr_t)newval);
}

static inline void *atomic_compare_exchange_global(void **ptr, void *oldval, void *newval) {
    return (void *)atomic_compare_exchange_global((intptr_t *)ptr, (intptr_t)oldval, (intptr_t)newval);
}

///////////////////////////////////////////////////////////////////////////
// local atomics

#define LOCAL_ATOMIC(TYPE, NAME, OPFUNC)                                                                               \
    static inline uniform TYPE atomic_##NAME##_local(uniform TYPE *uniform ptr, uniform TYPE value) {                  \
        uniform TYPE ret = *ptr;                                                                                       \
        *ptr = OPFUNC(*ptr, value);                                                                                    \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TYPE atomic_##NAME##_local(uniform TYPE *uniform ptr, TYPE value) {                                  \
        TYPE ret;                                                                                                      \
        foreach_active(i) {                                                                                            \
            ret = insert(ret, i, *ptr);                                                                                \
            *ptr = OPFUNC(*ptr, extract(value, i));                                                                    \
        }                                                                                                              \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TYPE atomic_##NAME##_local(uniform TYPE *p, TYPE value) {                                            \
        TYPE ret;                                                                                                      \
        uniform TYPE *uniform ptrs[programCount];                                                                      \
        ptrs[programIndex] = p;                                                                                        \
        foreach_active(i) {                                                                                            \
            ret = insert(ret, i, *ptrs[i]);                                                                            \
            *ptrs[i] = OPFUNC(*ptrs[i], extract(value, i));                                                            \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

static inline uniform int32 __add(uniform int32 a, uniform int32 b) { return a + b; }
static inline uniform int32 __sub(uniform int32 a, uniform int32 b) { return a - b; }
static inline uniform int32 __and(uniform int32 a, uniform int32 b) { return a & b; }
static inline uniform int32 __or(uniform int32 a, uniform int32 b) { return a | b; }
static inline uniform int32 __xor(uniform int32 a, uniform int32 b) { return a ^ b; }
static inline uniform int32 __swap(uniform int32 a, uniform int32 b) { return b; }

static inline uniform unsigned int32 __add(uniform unsigned int32 a, uniform unsigned int32 b) { return a + b; }
static inline uniform unsigned int32 __sub(uniform unsigned int32 a, uniform unsigned int32 b) { return a - b; }
static inline uniform unsigned int32 __and(uniform unsigned int32 a, uniform unsigned int32 b) { return a & b; }
static inline uniform unsigned int32 __or(uniform unsigned int32 a, uniform unsigned int32 b) { return a | b; }
static inline uniform unsigned int32 __xor(uniform unsigned int32 a, uniform unsigned int32 b) { return a ^ b; }
static inline uniform unsigned int32 __swap(uniform unsigned int32 a, uniform unsigned int32 b) { return b; }

static inline uniform float __add(uniform float a, uniform float b) { return a + b; }
static inline uniform float __sub(uniform float a, uniform float b) { return a - b; }
static inline uniform float __swap(uniform float a, uniform float b) { return b; }

static inline uniform int64 __add(uniform int64 a, uniform int64 b) { return a + b; }
static inline uniform int64 __sub(uniform int64 a, uniform int64 b) { return a - b; }
static inline uniform int64 __and(uniform int64 a, uniform int64 b) { return a & b; }
static inline uniform int64 __or(uniform int64 a, uniform int64 b) { return a | b; }
static inline uniform int64 __xor(uniform int64 a, uniform int64 b) { return a ^ b; }
static inline uniform int64 __swap(uniform int64 a, uniform int64 b) { return b; }

static inline uniform unsigned int64 __add(uniform unsigned int64 a, uniform unsigned int64 b) { return a + b; }
static inline uniform unsigned int64 __sub(uniform unsigned int64 a, uniform unsigned int64 b) { return a - b; }
static inline uniform unsigned int64 __and(uniform unsigned int64 a, uniform unsigned int64 b) { return a & b; }
static inline uniform unsigned int64 __or(uniform unsigned int64 a, uniform unsigned int64 b) { return a | b; }
static inline uniform unsigned int64 __xor(uniform unsigned int64 a, uniform unsigned int64 b) { return a ^ b; }
static inline uniform unsigned int64 __swap(uniform unsigned int64 a, uniform unsigned int64 b) { return b; }

static inline uniform double __add(uniform double a, uniform double b) { return a + b; }
static inline uniform double __sub(uniform double a, uniform double b) { return a - b; }
static inline uniform double __swap(uniform double a, uniform double b) { return a - b; }

LOCAL_ATOMIC(int32, add, __add)
LOCAL_ATOMIC(int32, subtract, __sub)
LOCAL_ATOMIC(int32, and, __and)
LOCAL_ATOMIC(int32, or, __or)
LOCAL_ATOMIC(int32, xor, __xor)
LOCAL_ATOMIC(int32, min, min)
LOCAL_ATOMIC(int32, max, max)
LOCAL_ATOMIC(int32, swap, __swap)

LOCAL_ATOMIC(unsigned int32, add, __add)
LOCAL_ATOMIC(unsigned int32, subtract, __sub)
LOCAL_ATOMIC(unsigned int32, and, __and)
LOCAL_ATOMIC(unsigned int32, or, __or)
LOCAL_ATOMIC(unsigned int32, xor, __xor)
LOCAL_ATOMIC(unsigned int32, min, min)
LOCAL_ATOMIC(unsigned int32, max, max)
LOCAL_ATOMIC(unsigned int32, swap, __swap)

LOCAL_ATOMIC(float, add, __add)
LOCAL_ATOMIC(float, subtract, __sub)
LOCAL_ATOMIC(float, min, min)
LOCAL_ATOMIC(float, max, max)
LOCAL_ATOMIC(float, swap, __swap)

LOCAL_ATOMIC(int64, add, __add)
LOCAL_ATOMIC(int64, subtract, __sub)
LOCAL_ATOMIC(int64, and, __and)
LOCAL_ATOMIC(int64, or, __or)
LOCAL_ATOMIC(int64, xor, __xor)
LOCAL_ATOMIC(int64, min, min)
LOCAL_ATOMIC(int64, max, max)
LOCAL_ATOMIC(int64, swap, __swap)

LOCAL_ATOMIC(unsigned int64, add, __add)
LOCAL_ATOMIC(unsigned int64, subtract, __sub)
LOCAL_ATOMIC(unsigned int64, and, __and)
LOCAL_ATOMIC(unsigned int64, or, __or)
LOCAL_ATOMIC(unsigned int64, xor, __xor)
LOCAL_ATOMIC(unsigned int64, min, min)
LOCAL_ATOMIC(unsigned int64, max, max)
LOCAL_ATOMIC(unsigned int64, swap, __swap)

LOCAL_ATOMIC(double, add, __add)
LOCAL_ATOMIC(double, subtract, __sub)
LOCAL_ATOMIC(double, min, min)
LOCAL_ATOMIC(double, max, max)
LOCAL_ATOMIC(double, swap, __swap)

// compare exchange
#define LOCAL_CMPXCHG(TYPE)                                                                                            \
    static inline uniform TYPE atomic_compare_exchange_local(uniform TYPE *uniform ptr, uniform TYPE cmp,              \
                                                             uniform TYPE update) {                                    \
        uniform TYPE old = *ptr;                                                                                       \
        if (old == cmp)                                                                                                \
            *ptr = update;                                                                                             \
        return old;                                                                                                    \
    }                                                                                                                  \
    static inline TYPE atomic_compare_exchange_local(uniform TYPE *uniform ptr, TYPE cmp, TYPE update) {               \
        TYPE ret;                                                                                                      \
        foreach_active(i) {                                                                                            \
            uniform TYPE old = *ptr;                                                                                   \
            if (old == extract(cmp, i))                                                                                \
                *ptr = extract(update, i);                                                                             \
            ret = insert(ret, i, old);                                                                                 \
        }                                                                                                              \
        return ret;                                                                                                    \
    }                                                                                                                  \
    static inline TYPE atomic_compare_exchange_local(uniform TYPE *varying p, TYPE cmp, TYPE update) {                 \
        uniform TYPE *uniform ptrs[programCount];                                                                      \
        ptrs[programIndex] = p;                                                                                        \
        TYPE ret;                                                                                                      \
        foreach_active(i) {                                                                                            \
            uniform TYPE old = *ptrs[i];                                                                               \
            if (old == extract(cmp, i))                                                                                \
                *ptrs[i] = extract(update, i);                                                                         \
            ret = insert(ret, i, old);                                                                                 \
        }                                                                                                              \
        return ret;                                                                                                    \
    }

LOCAL_CMPXCHG(int32)
LOCAL_CMPXCHG(unsigned int32)
LOCAL_CMPXCHG(float)
LOCAL_CMPXCHG(int64)
LOCAL_CMPXCHG(unsigned int64)
LOCAL_CMPXCHG(double)

#undef LOCAL_ATOMIC
#undef LOCAL_CMPXCHG

// void * variants of swap and compare exchange

static inline void *atomic_swap_local(void **uniform ptr, void *value) {
    return (void *)atomic_swap_local((uniform intptr_t * uniform) ptr, (intptr_t)value);
}

static inline void *uniform atomic_swap_local(void **uniform ptr, void *uniform value) {
    return (void *uniform)atomic_swap_local((uniform intptr_t * uniform) ptr, (uniform intptr_t)value);
}

static inline void *atomic_swap_local(void **ptr, void *value) {
    return (void *)atomic_swap_local((intptr_t *)ptr, (intptr_t)value);
}

static inline void *atomic_compare_exchange_local(void **uniform ptr, void *oldval, void *newval) {
    return (void *)atomic_compare_exchange_local((uniform intptr_t * uniform) ptr, (intptr_t)oldval, (intptr_t)newval);
}

static inline void *uniform atomic_compare_exchange_local(void **uniform ptr, void *uniform oldval,
                                                          void *uniform newval) {
    return (void *uniform)atomic_compare_exchange_local((uniform intptr_t * uniform) ptr, (uniform intptr_t)oldval,
                                                        (uniform intptr_t)newval);
}

static inline void *atomic_compare_exchange_local(void **ptr, void *oldval, void *newval) {
    return (void *)atomic_compare_exchange_local((intptr_t *)ptr, (intptr_t)oldval, (intptr_t)newval);
}

// Transcendentals (float precision)

__declspec(safe) static inline float sqrt(float v) {
    if (__math_lib == __math_lib_svml) {
        return __svml_sqrtf(v);
    } else {
        return __sqrt_varying_float(v);
    }
}

__declspec(safe) static inline uniform float sqrt(uniform float v) { return __sqrt_uniform_float(v); }

__declspec(safe) static inline float rsqrt(float v) {
    if (__math_lib == __math_lib_svml) {
        return __svml_invsqrtf(v);
    } else {
        return __rsqrt_varying_float(v);
    }
}

__declspec(safe) static inline uniform float rsqrt(uniform float v) { return __rsqrt_uniform_float(v); }

__declspec(safe) static inline float rsqrt_fast(float v) { return __rsqrt_fast_varying_float(v); }

__declspec(safe) static inline uniform float rsqrt_fast(uniform float v) { return __rsqrt_fast_uniform_float(v); }

__declspec(safe) static inline float ldexp(float x, int n) {
    varying unsigned int ex = 0x7F800000u;
    varying unsigned int ix = intbits(x);
    ex &= ix; // extract old exponent;
    float res = floatbits((ix & ~0x7F800000u) | ((n << 23) + ex));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0f) ? x : res;
    }
}

__declspec(safe) static inline uniform float ldexp(uniform float x, uniform int n) {
    uniform unsigned int ex = 0x7F800000u;
    uniform unsigned int ix = intbits(x);
    ex &= ix; // extract old exponent;
    uniform float res = floatbits((ix & ~0x7F800000u) | ((n << 23) + ex));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0f) ? x : res;
    }
}

__declspec(safe) static inline float frexp(float x, varying int *uniform pw2) {
    unsigned int ex = 0x7F800000u; // exponent mask
    unsigned int ix = intbits(x);
    ex &= ix;
    ix &= ~0x7F800000u;           // clear exponent
    *pw2 = (int)(ex >> 23) - 126; // compute exponent
    ix |= 0x3F000000u;            // insert exponent +1 in x
    return floatbits(ix);
}

__declspec(safe) static inline uniform float frexp(uniform float x, uniform int *uniform pw2) {
    uniform unsigned int ex = 0x7F800000u; // exponent mask
    uniform unsigned int ix = intbits(x);
    ex &= ix;
    ix &= ~0x7F800000u;                   // clear exponent
    *pw2 = (uniform int)(ex >> 23) - 126; // compute exponent
    ix |= 0x3F000000u;                    // insert exponent +1 in x
    return floatbits(ix);
}

// Most of the transcendental implementations in ispc code here come from
// Solomon Boulos's "syrah": https://github.com/boulos/syrah/

__declspec(safe) static inline float sin(float x_full) {
    if (__have_native_trigonometry) {
        return __sin_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_sinf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_sinf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        static const float pi_over_two_vec = 1.57079637050628662109375;
        static const float two_over_pi_vec = 0.636619746685028076171875;
        float scaled = x_full * two_over_pi_vec;
        float k_real = floor(scaled);
        int k = (int)k_real;

        // Reduced range version of x
        float x = x_full - k_real * pi_over_two_vec;
        int k_mod4 = k & 3;
        bool sin_usecos = (k_mod4 == 1 || k_mod4 == 3);
        bool flip_sign = (k_mod4 > 1);

        // These coefficients are from sollya with fpminimax(sin(x)/x, [|0, 2,
        // 4, 6, 8, 10|], [|single...|], [0;Pi/2]);
        static const float sin_c2 = -0.16666667163372039794921875;
        static const float sin_c4 = 8.333347737789154052734375e-3;
        static const float sin_c6 = -1.9842604524455964565277099609375e-4;
        static const float sin_c8 = 2.760012648650445044040679931640625e-6;
        static const float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        static const float cos_c2 = -0.5;
        static const float cos_c4 = 4.166664183139801025390625e-2;
        static const float cos_c6 = -1.388833043165504932403564453125e-3;
        static const float cos_c8 = 2.47562347794882953166961669921875e-5;
        static const float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        float outside = sin_usecos ? 1 : x;
        float c2 = sin_usecos ? cos_c2 : sin_c2;
        float c4 = sin_usecos ? cos_c4 : sin_c4;
        float c6 = sin_usecos ? cos_c6 : sin_c6;
        float c8 = sin_usecos ? cos_c8 : sin_c8;
        float c10 = sin_usecos ? cos_c10 : sin_c10;

        float x2 = x * x;
        float formula = x2 * c10 + c8;
        formula = x2 * formula + c6;
        formula = x2 * formula + c4;
        formula = x2 * formula + c2;
        formula = x2 * formula + 1;
        formula *= outside;

        formula = flip_sign ? -formula : formula;
        return formula;
    }
}

__declspec(safe) static inline uniform float sin(uniform float x_full) {
    if (__have_native_trigonometry) {
        return __sin_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_sinf(x_full);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        static const uniform float pi_over_two_vec = 1.57079637050628662109375;
        static const uniform float two_over_pi_vec = 0.636619746685028076171875;
        uniform float scaled = x_full * two_over_pi_vec;
        uniform float k_real = floor(scaled);
        uniform int k = (int)k_real;

        // Reduced range version of x
        uniform float x = x_full - k_real * pi_over_two_vec;
        uniform int k_mod4 = k & 3;
        uniform bool sin_usecos = (k_mod4 == 1 || k_mod4 == 3);
        uniform bool flip_sign = (k_mod4 > 1);

        // These coefficients are from sollya with fpminimax(sin(x)/x, [|0, 2,
        // 4, 6, 8, 10|], [|single...|], [0;Pi/2]);
        static const uniform float sin_c2 = -0.16666667163372039794921875;
        static const uniform float sin_c4 = 8.333347737789154052734375e-3;
        static const uniform float sin_c6 = -1.9842604524455964565277099609375e-4;
        static const uniform float sin_c8 = 2.760012648650445044040679931640625e-6;
        static const uniform float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        static const uniform float cos_c2 = -0.5;
        static const uniform float cos_c4 = 4.166664183139801025390625e-2;
        static const uniform float cos_c6 = -1.388833043165504932403564453125e-3;
        static const uniform float cos_c8 = 2.47562347794882953166961669921875e-5;
        static const uniform float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        uniform float outside, c2, c4, c6, c8, c10;
        if (sin_usecos) {
            outside = 1.;
            c2 = cos_c2;
            c4 = cos_c4;
            c6 = cos_c6;
            c8 = cos_c8;
            c10 = cos_c10;
        } else {
            outside = x;
            c2 = sin_c2;
            c4 = sin_c4;
            c6 = sin_c6;
            c8 = sin_c8;
            c10 = sin_c10;
        }

        uniform float x2 = x * x;
        uniform float formula = x2 * c10 + c8;
        formula = x2 * formula + c6;
        formula = x2 * formula + c4;
        formula = x2 * formula + c2;
        formula = x2 * formula + 1.;
        formula *= outside;

        formula = flip_sign ? -formula : formula;
        return formula;
    }
}

__declspec(safe) static inline uniform float sinh(uniform float x) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_sinhf(x);
    } else if (__math_lib == __math_lib_ispc_fast || __math_lib == __math_lib_ispc) {
        // Precision dependent to `expm1` (just slighly bigger).
        // Currently <5 ULP in default-math-lib.
        return 0.5 * (expm1(x) - expm1(-x));
    }
}

__declspec(safe) static inline float sinh(float x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_sinhf(x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_sinhf(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast || __math_lib == __math_lib_ispc) {
        // See the uniform version for more information
        return 0.5 * (expm1(x) - expm1(-x));
    }
}

__declspec(safe) static inline float asin(float x0) {

    // See the uniform implementation for more information

    const bool isneg = x0 < 0;
    const float x = abs(x0);
    const float y = x0 * x0 + 1e-30;
    const bool isnan = x > 1;
    float v, w;

    if (__have_native_trigonometry && !__is_xe_target) {
        return __asin_varying_float(x0);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_asinf(x0);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_asinf(extract(x0, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc) {
        v = 1.57035851f + x * (-0.210944608f + x * (0.07633722f + x * (-0.02661739f + x * 0.0050822f)));
        v = 1.570796371f - sqrt(1.f - x) * v;
        v = select(isneg, -v, v);
        v = select(isnan, floatbits(0x7fc00000), v);
        w = 1.0f + y * (0.1666585f + y * (0.075334f + y * (0.040581f + y * 0.04868f)));
        v = select(x < 0.492f, w * x0, v);
    } else if (__math_lib == __math_lib_ispc_fast) {
        v = +1.570578683e+00f +
            x * (-2.123403407e-01f + x * (+7.958955448e-02f + x * (-2.992102928e-02f + x * +6.318030601e-03f)));
        v = 1.570796371f - sqrt(1.f - x) * v;
        v = select(isneg, -v, v);
        v = select(isnan, floatbits(0x7fc00000), v);
        v = select(x < 0.30f, (1.0f + (0.166493f + 0.08106f * y) * y) * x0, v);
    }

    return v;
}

__declspec(safe) static inline uniform float asin(uniform float x0) {

    // Relative precision:
    // - default math-lib: <5 ULP
    // - fast math-lib:    <30 ULP
    // Support special input values like NaN, -Inf, +Inf and subnormal numbers correctly.

    // Note the polynomials approximate (asin(x)-pi/2)/-sqrt(1-x).
    // There are 2 polynomials:
    // - one for input values close to 0 based on Taylor expension so to get a good relative error;
    // - one for bigger (positive) values.
    // Coefficients can be found with `curve_fit` from `scipy.optimize` in Python.

    // Note the 1e-30 can strongly impact performance
    // (it certainly avoid the generation of subnormals numbers).

    const uniform bool isneg = x0 < 0;
    const uniform float x = abs(x0);
    const uniform float y = x0 * x0 + 1e-30;
    const uniform bool isnan = x > 1;
    uniform float v, w;

    if (__have_native_trigonometry && !__is_xe_target) {
        return __asin_uniform_float(x0);
    } else if (__math_lib == __math_lib_svml || __math_lib == __math_lib_system) {
        return __stdlib_asinf(x0);
    } else if (__math_lib == __math_lib_ispc) {
        v = 1.57035851f + x * (-0.210944608f + x * (0.07633722f + x * (-0.02661739f + x * 0.0050822f)));
        v = 1.570796371f - sqrt(1.f - x) * v;
        v = select(isneg, -v, v);
        v = select(isnan, floatbits(0x7fc00000), v);
        w = 1.0f + y * (0.1666585f + y * (0.075334f + y * (0.040581f + y * 0.04868f)));
        v = select(x < 0.492f, w * x0, v);
    } else if (__math_lib == __math_lib_ispc_fast) {
        v = +1.570578683e+00f +
            x * (-2.123403407e-01f + x * (+7.958955448e-02f + x * (-2.992102928e-02f + x * +6.318030601e-03f)));
        v = 1.570796371f - sqrt(1.f - x) * v;
        v = select(isneg, -v, v);
        v = select(isnan, floatbits(0x7fc00000), v);
        v = select(x < 0.30f, (1.0f + (0.166493f + 0.08106f * y) * y) * x0, v);
    }

    return v;
}

__declspec(safe) static inline float cos(float x_full) {
    if (__have_native_trigonometry) {
        return __cos_varying_float(x_full);
    }
    if (__math_lib == __math_lib_svml) {
        return __svml_cosf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_cosf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        static const float pi_over_two_vec = 1.57079637050628662109375;
        static const float two_over_pi_vec = 0.636619746685028076171875;
        float scaled = x_full * two_over_pi_vec;
        float k_real = floor(scaled);
        int k = (int)k_real;

        // Reduced range version of x
        float x = x_full - k_real * pi_over_two_vec;

        int k_mod4 = k & 3;
        bool cos_usecos = (k_mod4 == 0 || k_mod4 == 2);
        bool flip_sign = (k_mod4 == 1 || k_mod4 == 2);

        const float sin_c2 = -0.16666667163372039794921875;
        const float sin_c4 = 8.333347737789154052734375e-3;
        const float sin_c6 = -1.9842604524455964565277099609375e-4;
        const float sin_c8 = 2.760012648650445044040679931640625e-6;
        const float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        const float cos_c2 = -0.5;
        const float cos_c4 = 4.166664183139801025390625e-2;
        const float cos_c6 = -1.388833043165504932403564453125e-3;
        const float cos_c8 = 2.47562347794882953166961669921875e-5;
        const float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        float outside = cos_usecos ? 1. : x;
        float c2 = cos_usecos ? cos_c2 : sin_c2;
        float c4 = cos_usecos ? cos_c4 : sin_c4;
        float c6 = cos_usecos ? cos_c6 : sin_c6;
        float c8 = cos_usecos ? cos_c8 : sin_c8;
        float c10 = cos_usecos ? cos_c10 : sin_c10;

        float x2 = x * x;
        float formula = x2 * c10 + c8;
        formula = x2 * formula + c6;
        formula = x2 * formula + c4;
        formula = x2 * formula + c2;
        formula = x2 * formula + 1.;
        formula *= outside;

        formula = flip_sign ? -formula : formula;
        return formula;
    }
}

__declspec(safe) static inline uniform float cos(uniform float x_full) {
    if (__have_native_trigonometry) {
        return __cos_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_cosf(x_full);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        static const uniform float pi_over_two_vec = 1.57079637050628662109375;
        static const uniform float two_over_pi_vec = 0.636619746685028076171875;
        uniform float scaled = x_full * two_over_pi_vec;
        uniform float k_real = floor(scaled);
        uniform int k = (int)k_real;

        // Reduced range version of x
        uniform float x = x_full - k_real * pi_over_two_vec;

        uniform int k_mod4 = k & 3;
        uniform bool cos_usecos = (k_mod4 == 0 || k_mod4 == 2);
        uniform bool flip_sign = (k_mod4 == 1 || k_mod4 == 2);

        const uniform float sin_c2 = -0.16666667163372039794921875;
        const uniform float sin_c4 = 8.333347737789154052734375e-3;
        const uniform float sin_c6 = -1.9842604524455964565277099609375e-4;
        const uniform float sin_c8 = 2.760012648650445044040679931640625e-6;
        const uniform float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        const uniform float cos_c2 = -0.5;
        const uniform float cos_c4 = 4.166664183139801025390625e-2;
        const uniform float cos_c6 = -1.388833043165504932403564453125e-3;
        const uniform float cos_c8 = 2.47562347794882953166961669921875e-5;
        const uniform float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        uniform float outside, c2, c4, c6, c8, c10;
        if (cos_usecos) {
            outside = 1.;
            c2 = cos_c2;
            c4 = cos_c4;
            c6 = cos_c6;
            c8 = cos_c8;
            c10 = cos_c10;
        } else {
            outside = x;
            c2 = sin_c2;
            c4 = sin_c4;
            c6 = sin_c6;
            c8 = sin_c8;
            c10 = sin_c10;
        }

        uniform float x2 = x * x;
        uniform float formula = x2 * c10 + c8;
        formula = x2 * formula + c6;
        formula = x2 * formula + c4;
        formula = x2 * formula + c2;
        formula = x2 * formula + 1.;
        formula *= outside;

        formula = flip_sign ? -formula : formula;
        return formula;
    }
}

__declspec(safe) static inline uniform float cosh(uniform float x) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_coshf(x);
    } else if (__math_lib == __math_lib_ispc_fast || __math_lib == __math_lib_ispc) {
        // Precision dependent to `exp` (slightly smaller).
        // Currently <5 ULP in default-math-lib.
        return 0.5 * (exp(x) + exp(-x));
    }
}

__declspec(safe) static inline float cosh(float x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_coshf(x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_coshf(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast || __math_lib == __math_lib_ispc) {
        // See the uniform version for more information
        return 0.5 * (exp(x) + exp(-x));
    }
}

__declspec(safe) static inline uniform float acos(uniform float x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __acos_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_acos(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <15 ULP.
        // Natively support special values (e.g. subnormal numbers and NaNs).

        // Sollya:
        // f = acos(1-x) / sqrt(x)
        // fpminimax(f, [|0,...,5|], [|single...|], [1e-300;1.0]);
        const uniform float c0 = +1.4142124653e+00;
        const uniform float c1 = +1.1792322248e-01;
        const uniform float c2 = +2.5741405785e-02;
        const uniform float c3 = +1.0879410430e-02;
        const uniform float c4 = -2.2721362770e-03;
        const uniform float c5 = +4.3107867240e-03;

        const uniform float x = abs(x_full);
        const uniform float t = 1.0 - x;
        const uniform float t2 = t * t;

        // Estrin's scheme used to reduce the latency
        uniform float poly = c5 * t + c4;
        poly = poly * t2 + (c3 * t + c2);
        poly = poly * t2 + (c1 * t + c0);
        poly *= sqrt(t);

        return select(x_full < 0.0, PI - poly, poly);
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <5 ULP

        // Sollya:
        // f = acos(1-x) / sqrt(x)
        // fpminimax(f, [|0,...,6|], [|single...|], [1e-300;1.004]);
        const uniform float c0 = +1.4142136574e+00;
        const uniform float c1 = +1.1783879995e-01;
        const uniform float c2 = +2.6711430400e-02;
        const uniform float c3 = +6.7900302820e-03;
        const uniform float c4 = +5.5398275140e-03;
        const uniform float c5 = -2.5801914740e-03;
        const uniform float c6 = +2.2826674390e-03;

        const uniform float x = abs(x_full);
        const uniform float t = 1.0 - x;
        const uniform float t2 = t * t;

        // Estrin's scheme used to reduce the latency
        uniform float poly = c6;
        poly = poly * t2 + (c5 * t + c4);
        poly = poly * t2 + (c3 * t + c2);
        poly = poly * t2 + (c1 * t + c0);
        poly *= sqrt(t);

        return select(x_full < 0.0, PI - poly, poly);
    }
}

__declspec(safe) static inline float acos(float x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __acos_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_acosf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_acosf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        const float c0 = +1.4142124653e+00;
        const float c1 = +1.1792322248e-01;
        const float c2 = +2.5741405785e-02;
        const float c3 = +1.0879410430e-02;
        const float c4 = -2.2721362770e-03;
        const float c5 = +4.3107867240e-03;

        const float x = abs(x_full);
        const float t = 1.0 - x;
        const float t2 = t * t;

        float poly = c5 * t + c4;
        poly = poly * t2 + (c3 * t + c2);
        poly = poly * t2 + (c1 * t + c0);
        poly *= sqrt(t);

        return select(x_full < 0.0, PI - poly, poly);
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        const float c0 = +1.4142136574e+00;
        const float c1 = +1.1783879995e-01;
        const float c2 = +2.6711430400e-02;
        const float c3 = +6.7900302820e-03;
        const float c4 = +5.5398275140e-03;
        const float c5 = -2.5801914740e-03;
        const float c6 = +2.2826674390e-03;

        const float x = abs(x_full);
        const float t = 1.0 - x;
        const float t2 = t * t;

        float poly = c6;
        poly = poly * t2 + (c5 * t + c4);
        poly = poly * t2 + (c3 * t + c2);
        poly = poly * t2 + (c1 * t + c0);
        poly *= sqrt(t);

        return select(x_full < 0.0, PI - poly, poly);
    }
}

__declspec(safe) static inline void sincos(float x_full, varying float *uniform sin_result,
                                           varying float *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_varying_float(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
        return;
    }
    if (__math_lib == __math_lib_svml) {
        __svml_sincosf(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else if (__math_lib == __math_lib_system) {
        foreach_active(i) {
            uniform float s, c;
            __stdlib_sincosf(extract(x_full, i), (opaque_ptr_t)&s, (opaque_ptr_t)&c);
            *sin_result = insert(*sin_result, i, s);
            *cos_result = insert(*cos_result, i, c);
        }
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const float pi_over_two_vec = 1.57079637050628662109375;
        const float two_over_pi_vec = 0.636619746685028076171875;
        float scaled = x_full * two_over_pi_vec;
        float k_real = floor(scaled);
        int k = (int)k_real;

        // Reduced range version of x
        float x = x_full - k_real * pi_over_two_vec;
        int k_mod4 = k & 3;
        bool cos_usecos = (k_mod4 == 0 || k_mod4 == 2);
        bool sin_usecos = (k_mod4 == 1 || k_mod4 == 3);
        bool sin_flipsign = (k_mod4 > 1);
        bool cos_flipsign = (k_mod4 == 1 || k_mod4 == 2);

        const float one_vec = 1.;
        const float sin_c2 = -0.16666667163372039794921875;
        const float sin_c4 = 8.333347737789154052734375e-3;
        const float sin_c6 = -1.9842604524455964565277099609375e-4;
        const float sin_c8 = 2.760012648650445044040679931640625e-6;
        const float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        const float cos_c2 = -0.5;
        const float cos_c4 = 4.166664183139801025390625e-2;
        const float cos_c6 = -1.388833043165504932403564453125e-3;
        const float cos_c8 = 2.47562347794882953166961669921875e-5;
        const float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        float x2 = x * x;

        float sin_formula = x2 * sin_c10 + sin_c8;
        float cos_formula = x2 * cos_c10 + cos_c8;
        sin_formula = x2 * sin_formula + sin_c6;
        cos_formula = x2 * cos_formula + cos_c6;

        sin_formula = x2 * sin_formula + sin_c4;
        cos_formula = x2 * cos_formula + cos_c4;

        sin_formula = x2 * sin_formula + sin_c2;
        cos_formula = x2 * cos_formula + cos_c2;

        sin_formula = x2 * sin_formula + one_vec;
        cos_formula = x2 * cos_formula + one_vec;

        sin_formula *= x;

        *sin_result = sin_usecos ? cos_formula : sin_formula;
        *cos_result = cos_usecos ? cos_formula : sin_formula;

        *sin_result = sin_flipsign ? -*sin_result : *sin_result;
        *cos_result = cos_flipsign ? -*cos_result : *cos_result;
    }
}

__declspec(safe) static inline void sincos(uniform float x_full, uniform float *uniform sin_result,
                                           uniform float *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_uniform_float(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
        return;
    }
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        __stdlib_sincosf(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const uniform float pi_over_two_vec = 1.57079637050628662109375;
        const uniform float two_over_pi_vec = 0.636619746685028076171875;
        uniform float scaled = x_full * two_over_pi_vec;
        uniform float k_real = floor(scaled);
        uniform int k = (uniform int)k_real;

        // Reduced range version of x
        uniform float x = x_full - k_real * pi_over_two_vec;
        uniform int k_mod4 = k & 3;
        uniform bool cos_usecos = (k_mod4 == 0 || k_mod4 == 2);
        uniform bool sin_usecos = (k_mod4 == 1 || k_mod4 == 3);
        uniform bool sin_flipsign = (k_mod4 > 1);
        uniform bool cos_flipsign = (k_mod4 == 1 || k_mod4 == 2);

        const uniform float one_vec = 1.;
        const uniform float sin_c2 = -0.16666667163372039794921875;
        const uniform float sin_c4 = 8.333347737789154052734375e-3;
        const uniform float sin_c6 = -1.9842604524455964565277099609375e-4;
        const uniform float sin_c8 = 2.760012648650445044040679931640625e-6;
        const uniform float sin_c10 = -2.50293279435709337121807038784027099609375e-8;

        const uniform float cos_c2 = -0.5;
        const uniform float cos_c4 = 4.166664183139801025390625e-2;
        const uniform float cos_c6 = -1.388833043165504932403564453125e-3;
        const uniform float cos_c8 = 2.47562347794882953166961669921875e-5;
        const uniform float cos_c10 = -2.59630184018533327616751194000244140625e-7;

        uniform float x2 = x * x;

        uniform float sin_formula = x2 * sin_c10 + sin_c8;
        uniform float cos_formula = x2 * cos_c10 + cos_c8;
        sin_formula = x2 * sin_formula + sin_c6;
        cos_formula = x2 * cos_formula + cos_c6;

        sin_formula = x2 * sin_formula + sin_c4;
        cos_formula = x2 * cos_formula + cos_c4;

        sin_formula = x2 * sin_formula + sin_c2;
        cos_formula = x2 * cos_formula + cos_c2;

        sin_formula = x2 * sin_formula + one_vec;
        cos_formula = x2 * cos_formula + one_vec;

        sin_formula *= x;

        *sin_result = sin_usecos ? cos_formula : sin_formula;
        *cos_result = cos_usecos ? cos_formula : sin_formula;

        *sin_result = sin_flipsign ? -*sin_result : *sin_result;
        *cos_result = cos_flipsign ? -*cos_result : *cos_result;
    }
}

__declspec(safe) static inline float tan(float x_full) {
    if (__have_native_trigonometry) {
        return __tan_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_tanf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_tanf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const float pi_over_four_vec = 0.785398185253143310546875;
        const float four_over_pi_vec = 1.27323949337005615234375;

        bool x_lt_0 = x_full < 0.;
        float y = x_lt_0 ? -x_full : x_full;
        float scaled = y * four_over_pi_vec;

        float k_real = floor(scaled);
        int k = (int)k_real;

        float x = y - k_real * pi_over_four_vec;

        // if k & 1, x -= Pi/4
        bool need_offset = (k & 1) != 0;
        x = need_offset ? x - pi_over_four_vec : x;

        // if k & 3 == (0 or 3) let z = tan_In...(y) otherwise z = -cot_In0To...
        int k_mod4 = k & 3;
        bool use_cotan = (k_mod4 == 1) || (k_mod4 == 2);

        const float one_vec = 1.0;

        const float tan_c2 = 0.33333075046539306640625;
        const float tan_c4 = 0.13339905440807342529296875;
        const float tan_c6 = 5.3348250687122344970703125e-2;
        const float tan_c8 = 2.46033705770969390869140625e-2;
        const float tan_c10 = 2.892402000725269317626953125e-3;
        const float tan_c12 = 9.500005282461643218994140625e-3;

        const float cot_c2 = -0.3333333432674407958984375;
        const float cot_c4 = -2.222204394638538360595703125e-2;
        const float cot_c6 = -2.11752182804048061370849609375e-3;
        const float cot_c8 = -2.0846328698098659515380859375e-4;
        const float cot_c10 = -2.548247357481159269809722900390625e-5;
        const float cot_c12 = -3.5257363606433500535786151885986328125e-7;

        float x2 = x * x;
        float z;
        cif(use_cotan) {
            float cot_val = x2 * cot_c12 + cot_c10;
            cot_val = x2 * cot_val + cot_c8;
            cot_val = x2 * cot_val + cot_c6;
            cot_val = x2 * cot_val + cot_c4;
            cot_val = x2 * cot_val + cot_c2;
            cot_val = x2 * cot_val + one_vec;
            // The equation is for x * cot(x) but we need -x * cot(x) for the tan part.
            cot_val /= -x;
            z = cot_val;
        }
        else {
            float tan_val = x2 * tan_c12 + tan_c10;
            tan_val = x2 * tan_val + tan_c8;
            tan_val = x2 * tan_val + tan_c6;
            tan_val = x2 * tan_val + tan_c4;
            tan_val = x2 * tan_val + tan_c2;
            tan_val = x2 * tan_val + one_vec;
            // Equation was for tan(x)/x
            tan_val *= x;
            z = tan_val;
        }
        return x_lt_0 ? -z : z;
    }
}

__declspec(safe) static inline uniform float tan(uniform float x_full) {
    if (__have_native_trigonometry) {
        return __tan_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_tanf(x_full);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const uniform float pi_over_four_vec = 0.785398185253143310546875;
        const uniform float four_over_pi_vec = 1.27323949337005615234375;

        uniform bool x_lt_0 = x_full < 0.;
        uniform float y = x_lt_0 ? -x_full : x_full;
        uniform float scaled = y * four_over_pi_vec;

        uniform float k_real = floor(scaled);
        uniform int k = (int)k_real;

        uniform float x = y - k_real * pi_over_four_vec;

        // if k & 1, x -= Pi/4
        uniform bool need_offset = (k & 1) != 0;
        x = need_offset ? x - pi_over_four_vec : x;

        // if k & 3 == (0 or 3) let z = tan_In...(y) otherwise z = -cot_In0To...
        uniform int k_mod4 = k & 3;
        uniform bool use_cotan = (k_mod4 == 1) || (k_mod4 == 2);

        const uniform float one_vec = 1.0;

        const uniform float tan_c2 = 0.33333075046539306640625;
        const uniform float tan_c4 = 0.13339905440807342529296875;
        const uniform float tan_c6 = 5.3348250687122344970703125e-2;
        const uniform float tan_c8 = 2.46033705770969390869140625e-2;
        const uniform float tan_c10 = 2.892402000725269317626953125e-3;
        const uniform float tan_c12 = 9.500005282461643218994140625e-3;

        const uniform float cot_c2 = -0.3333333432674407958984375;
        const uniform float cot_c4 = -2.222204394638538360595703125e-2;
        const uniform float cot_c6 = -2.11752182804048061370849609375e-3;
        const uniform float cot_c8 = -2.0846328698098659515380859375e-4;
        const uniform float cot_c10 = -2.548247357481159269809722900390625e-5;
        const uniform float cot_c12 = -3.5257363606433500535786151885986328125e-7;

        uniform float x2 = x * x;
        uniform float z;
        if (use_cotan) {
            uniform float cot_val = x2 * cot_c12 + cot_c10;
            cot_val = x2 * cot_val + cot_c8;
            cot_val = x2 * cot_val + cot_c6;
            cot_val = x2 * cot_val + cot_c4;
            cot_val = x2 * cot_val + cot_c2;
            cot_val = x2 * cot_val + one_vec;
            // The equation is for x * cot(x) but we need -x * cot(x) for the tan part.
            cot_val /= -x;
            z = cot_val;
        } else {
            uniform float tan_val = x2 * tan_c12 + tan_c10;
            tan_val = x2 * tan_val + tan_c8;
            tan_val = x2 * tan_val + tan_c6;
            tan_val = x2 * tan_val + tan_c4;
            tan_val = x2 * tan_val + tan_c2;
            tan_val = x2 * tan_val + one_vec;
            // Equation was for tan(x)/x
            tan_val *= x;
            z = tan_val;
        }
        return x_lt_0 ? -z : z;
    }
}

__declspec(safe) static inline uniform float atan(uniform float x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_atanf(x_full);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {

        // Precision:
        //  - Default version: <5 ULP
        //  - Fast version: <15 ULP
        // All numbers are properly supported in the fast implementation.

        // Implementation notes:
        // - `atan(-x) = -atan(x)` (so flip from negative to positive first)
        // - if x > 1 -> atan(x) = Pi/2 - atan(1/x)
        // - A tiny epsilon is added so to compute subnormal numbers faster

        const uniform float pi_over_two_vec = 1.5707963705;
        uniform float x_abs = abs(x_full);
        uniform bool x_gt_1 = x_abs > 1.0;
        uniform float x = x_gt_1 ? 1.0 / x_abs : x_abs;
        uniform float x2 = x * x + 1e-30;
        uniform float result;

        if (__math_lib == __math_lib_ispc_fast) {
            // Sollya:
            // r = [0.0;1.0];
            // fpminimax(atan(x), [|1,3,5,7,9,11,13|], [|single...|], r);
            const uniform float atan_c00 = +9.9999934430e-01;
            const uniform float atan_c02 = -3.3326506610e-01;
            const uniform float atan_c04 = +1.9881419840e-01;
            const uniform float atan_c06 = -1.3486981390e-01;
            const uniform float atan_c08 = +8.3867706360e-02;
            const uniform float atan_c10 = -3.7010200320e-02;
            const uniform float atan_c12 = +7.8625064340e-03;

            uniform float poly = atan_c12;
            poly = x2 * poly + atan_c10;
            poly = x2 * poly + atan_c08;
            poly = x2 * poly + atan_c06;
            poly = x2 * poly + atan_c04;
            poly = x2 * poly + atan_c02;
            poly = x2 * poly + atan_c00;
            result = poly * x;
        } else {
            // Sollya:
            // r = [0.0;1.0];
            // fpminimax(atan(x), [|1,3,5,7,9,11,13,15|], [|single...|], r);
            const uniform float atan_c00 = +9.9999988079e-01;
            const uniform float atan_c02 = -3.3331915736e-01;
            const uniform float atan_c04 = +1.9968920946e-01;
            const uniform float atan_c06 = -1.4015688002e-01;
            const uniform float atan_c08 = +9.9050834775e-02;
            const uniform float atan_c10 = -5.9366498142e-02;
            const uniform float atan_c12 = +2.4172833189e-02;
            const uniform float atan_c14 = -4.6721356921e-03;

            uniform float poly = atan_c14;
            poly = x2 * poly + atan_c12;
            poly = x2 * poly + atan_c10;
            poly = x2 * poly + atan_c08;
            poly = x2 * poly + atan_c06;
            poly = x2 * poly + atan_c04;
            poly = x2 * poly + atan_c02;
            poly = x2 * poly + atan_c00;
            result = poly * x;
        }

        result = x_gt_1 ? pi_over_two_vec - result : result;

        // Transfer the sign of `x_full` to `result`
        return floatbits((intbits(x_full) & 0x80000000ul) ^ intbits(result));
    }
}

__declspec(safe) static inline float atan(float x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_atanf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_atanf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        const float pi_over_two_vec = 1.5707963705;
        float x_abs = abs(x_full);
        bool x_gt_1 = x_abs > 1.0;
        float x = x_gt_1 ? 1.0 / x_abs : x_abs;
        float x2 = x * x + 1e-30;
        float result;

        if (__math_lib == __math_lib_ispc_fast) {
            const float atan_c00 = +9.9999934430e-01;
            const float atan_c02 = -3.3326506610e-01;
            const float atan_c04 = +1.9881419840e-01;
            const float atan_c06 = -1.3486981390e-01;
            const float atan_c08 = +8.3867706360e-02;
            const float atan_c10 = -3.7010200320e-02;
            const float atan_c12 = +7.8625064340e-03;

            float poly = atan_c12;
            poly = x2 * poly + atan_c10;
            poly = x2 * poly + atan_c08;
            poly = x2 * poly + atan_c06;
            poly = x2 * poly + atan_c04;
            poly = x2 * poly + atan_c02;
            poly = x2 * poly + atan_c00;
            result = poly * x;
        } else {
            const float atan_c00 = +9.9999988079e-01;
            const float atan_c02 = -3.3331915736e-01;
            const float atan_c04 = +1.9968920946e-01;
            const float atan_c06 = -1.4015688002e-01;
            const float atan_c08 = +9.9050834775e-02;
            const float atan_c10 = -5.9366498142e-02;
            const float atan_c12 = +2.4172833189e-02;
            const float atan_c14 = -4.6721356921e-03;

            float poly = atan_c14;
            poly = x2 * poly + atan_c12;
            poly = x2 * poly + atan_c10;
            poly = x2 * poly + atan_c08;
            poly = x2 * poly + atan_c06;
            poly = x2 * poly + atan_c04;
            poly = x2 * poly + atan_c02;
            poly = x2 * poly + atan_c00;
            result = poly * x;
        }

        result = x_gt_1 ? pi_over_two_vec - result : result;
        return floatbits((intbits(x_full) & 0x80000000) ^ intbits(result));
    }
}

__declspec(safe) static inline float atan2(float y, float x) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan2_varying_float(y, x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_atan2f(y, x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_atan2f(extract(y, i), extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const float pi_vec = 3.1415926536;
        const float pi_over_two_vec = 1.5707963267;
        // atan2(y, x) =
        //
        // atan2(y > 0, x = +-0) ->  Pi/2
        // atan2(y < 0, x = +-0) -> -Pi/2
        // atan2(y = +-0, x < +0) -> +-Pi
        // atan2(y = +-0, x >= +0) -> +-0
        //
        // atan2(y >= 0, x < 0) ->  Pi + atan(y/x)
        // atan2(y <  0, x < 0) -> -Pi + atan(y/x)
        // atan2(y, x > 0) -> atan(y/x)
        //
        // and then a bunch of code for dealing with infinities.
        float y_over_x = y / x;
        float atan_arg = atan(y_over_x);
        bool x_lt_0 = x < 0;
        bool y_lt_0 = y < 0;
        float offset = x_lt_0 ? (y_lt_0 ? -pi_vec : pi_vec) : 0;
        return offset + atan_arg;
    }
}

__declspec(safe) static inline uniform float atan2(uniform float y, uniform float x) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan2_uniform_float(y, x);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_atan2f(y, x);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        const uniform float pi_vec = 3.1415927410125732421875;
        const uniform float pi_over_two_vec = 1.57079637050628662109375;

        uniform float y_over_x = y / x;
        uniform float atan_arg = atan(y_over_x);
        uniform bool x_lt_0 = x < 0;
        uniform bool y_lt_0 = y < 0;
        uniform float offset = x_lt_0 ? (y_lt_0 ? -pi_vec : pi_vec) : 0;
        return offset + atan_arg;
    }
}

__declspec(safe) static inline uniform float tanh(uniform float x) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_tanhf(x);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <10 ULP
        // Padé approximation generated from Lambert's continued fractions and
        // manually optimized (with Sollya) so to get a smaller maximum error
        // on the range [-8.0;8.0].
        // Properly support special values (e.g. NaN, Inf, subnormals).

        const uniform float a1 = +3.4459429000e+07;
        const uniform float a2 = +4.8225360000e+06;
        const uniform float a3 = +1.4751137500e+05;
        const uniform float a4 = +1.3072869873e+03;
        const uniform float a5 = +2.6487326620e+00;

        const uniform float b0 = +3.4459432000e+07;
        const uniform float b1 = +1.6308998000e+07;
        const uniform float b2 = +9.8926356250e+05;
        const uniform float b3 = +1.6239637695e+04;
        const uniform float b4 = +7.3853576660e+01;
        const uniform float b5 = +4.5743070540e-02;

        // See the default version regarding the tiny epsilon
        uniform float x2 = x * x + 1e-37;

        uniform float a_poly = a5;
        a_poly = x2 * a_poly + a4;
        a_poly = x2 * a_poly + a3;
        a_poly = x2 * a_poly + a2;
        a_poly = x2 * a_poly + a1;
        a_poly = x * a_poly;

        uniform float b_poly = b5;
        b_poly = x2 * b_poly + b4;
        b_poly = x2 * b_poly + b3;
        b_poly = x2 * b_poly + b2;
        b_poly = x2 * b_poly + b1;
        b_poly = x2 * b_poly + b0;

        uniform float res = a_poly / b_poly;
        return select(x < -8.0, -1.0, select(x > 8.0, 1.0, res));
    } else if (__math_lib == __math_lib_ispc) {

        // Precision:
        // - fixup range: <5 ULP
        // - other range: the error should be slighly higher than `exp`,
        //                in practice, it is <5 ULP with the current `exp`

        uniform float xt = select(x < -9.5, -9.5, x);
        uniform float t = exp(-xt);
        uniform float res = (t * t - 1.0) / (-1.0 - t * t);

        // Sollya:
        // fpminimax(tanh(x), [|3,5,7|], [|single...|], [-0.425;0.425], x);
        const uniform float c3 = -3.3331277970e-01;
        const uniform float c5 = +1.3263334334e-01;
        const uniform float c7 = -4.7071605920e-02;

        // Adding a tiny epsilon reduces the generation of subnormal numbers.
        // This massively improves the execution time when 1e-23 < x < 1e-19 on
        // CPUs computing subnormal numbers much slower (e.g. most Intel CPUs).
        uniform float x2 = x * x + 1e-37;
        uniform float fix = c7;
        fix = x2 * fix + c5;
        fix = x2 * fix + c3;
        fix = (x2 * x) * fix + x;

        return select(abs(x) < 0.425, fix, res);
    }
}

__declspec(safe) static inline float tanh(float x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_tanhf(x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_tanhf(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {
        const float a1 = +3.4459429000e+07;
        const float a2 = +4.8225360000e+06;
        const float a3 = +1.4751137500e+05;
        const float a4 = +1.3072869873e+03;
        const float a5 = +2.6487326620e+00;

        const float b0 = +3.4459432000e+07;
        const float b1 = +1.6308998000e+07;
        const float b2 = +9.8926356250e+05;
        const float b3 = +1.6239637695e+04;
        const float b4 = +7.3853576660e+01;
        const float b5 = +4.5743070540e-02;

        float x2 = x * x + 1e-37;

        float a_poly = a5;
        a_poly = x2 * a_poly + a4;
        a_poly = x2 * a_poly + a3;
        a_poly = x2 * a_poly + a2;
        a_poly = x2 * a_poly + a1;
        a_poly = x * a_poly;

        float b_poly = b5;
        b_poly = x2 * b_poly + b4;
        b_poly = x2 * b_poly + b3;
        b_poly = x2 * b_poly + b2;
        b_poly = x2 * b_poly + b1;
        b_poly = x2 * b_poly + b0;

        float res = a_poly / b_poly;
        return select(x < -8.0, -1.0, select(x > 8.0, 1.0, res));
    } else if (__math_lib == __math_lib_ispc) {
        float xt = select(x < -9.5, -9.5, x);
        float t = exp(-xt);
        float res = (t * t - 1.0) / (-1.0 - t * t);

        const float c3 = -3.3331277970e-01;
        const float c5 = +1.3263334334e-01;
        const float c7 = -4.7071605920e-02;

        float x2 = x * x + 1e-37;
        float fix = c7;
        fix = x2 * fix + c5;
        fix = x2 * fix + c3;
        fix = (x2 * x) * fix + x;

        return select(abs(x) < 0.425, fix, res);
    }
}

__declspec(safe) static inline float exp(float x_full) {
    if (__have_native_transcendentals) {
        return __exp_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_expf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_expf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {
        float z = floor(1.44269504088896341f * x_full + 0.5f);
        int n;
        x_full -= z * 0.693359375f;
        x_full -= z * -2.12194440e-4f;
        n = (int)z;

        z = x_full * x_full;
        z = (((((1.9875691500E-4f * x_full + 1.3981999507E-3f) * x_full + 8.3334519073E-3f) * x_full +
               4.1665795894E-2f) *
                  x_full +
              1.6666665459E-1f) *
                 x_full +
             5.0000001201E-1f) *
                z +
            x_full + 1.f;
        x_full = ldexp(z, n);
        return x_full;
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        const float ln2_part1 = 0.6931457519;
        const float ln2_part2 = 1.4286067653e-6;
        const float one_over_ln2 = 1.44269502162933349609375;

        float scaled = x_full * one_over_ln2;
        float k_real = floor(scaled);
        int k = (int)k_real;

        // Reduced range version of x
        float x = x_full - k_real * ln2_part1;
        x -= k_real * ln2_part2;

        // These coefficients are for e^x in [0, ln(2)]
        const float one = 1.;
        const float c2 = 0.4999999105930328369140625;
        const float c3 = 0.166668415069580078125;
        const float c4 = 4.16539050638675689697265625e-2;
        const float c5 = 8.378830738365650177001953125e-3;
        const float c6 = 1.304379315115511417388916015625e-3;
        const float c7 = 2.7555381529964506626129150390625e-4;

        float result = x * c7 + c6;
        result = x * result + c5;
        result = x * result + c4;
        result = x * result + c3;
        result = x * result + c2;
        result = x * result + one;
        result = x * result + one;

        // Compute 2^k (should differ for float and double, but I'll avoid
        // it for now and just do floats)
        const int fpbias = 127;
        int biased_n = k + fpbias;
        bool overflow = k_real > fpbias;
        // Minimum exponent is -126, so if k is <= -127 (k + 127 <= 0)
        // we've got underflow. -127 * ln(2) -> -88.02. So the most
        // negative float input that doesn't result in zero is like -88.
        bool underflow = k_real <= -fpbias;
        const int InfBits = 0x7f800000;
        biased_n <<= 23;
        // Reinterpret this thing as float
        float two_to_the_n = floatbits(biased_n);
        // Handle both doubles and floats (hopefully eliding the copy for float)
        float elemtype_2n = two_to_the_n;
        result *= elemtype_2n;
        result = overflow ? floatbits(InfBits) : result;
        result = underflow ? 0. : result;
        return result;
    }
}

__declspec(safe) static inline uniform float exp(uniform float x_full) {
    if (__have_native_transcendentals) {
        return __exp_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_expf(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {
        uniform float z = floor(1.44269504088896341f * x_full + 0.5f);
        uniform int n;
        x_full -= z * 0.693359375f;
        x_full -= z * -2.12194440e-4f;
        n = (int)z;

        z = x_full * x_full;
        z = (((((1.9875691500E-4f * x_full + 1.3981999507E-3f) * x_full + 8.3334519073E-3f) * x_full +
               4.1665795894E-2f) *
                  x_full +
              1.6666665459E-1f) *
                 x_full +
             5.0000001201E-1f) *
                z +
            x_full + 1.f;
        x_full = ldexp(z, n);
        return x_full;
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <5 ULP for normal numbers (possibly much higher for subnormal like exp(-87.34)).
        // Support special input values like NaN, -Inf, +Inf and subnormal numbers correctly.

        const uniform float ln2_part1 = 0.6931457519;
        const uniform float ln2_part2 = 1.4286067653e-6;
        const uniform float one_over_ln2 = 1.44269502162933349609375;

        uniform float scaled = x_full * one_over_ln2;
        uniform float k_real = floor(scaled);
        uniform int k = (uniform int)k_real;

        // Reduced range version of x
        uniform float x = x_full - k_real * ln2_part1;
        x -= k_real * ln2_part2;

        // These coefficients are for e^x in [0, ln(2)]
        const uniform float one = 1.;
        const uniform float c2 = 0.4999999105930328369140625;
        const uniform float c3 = 0.166668415069580078125;
        const uniform float c4 = 4.16539050638675689697265625e-2;
        const uniform float c5 = 8.378830738365650177001953125e-3;
        const uniform float c6 = 1.304379315115511417388916015625e-3;
        const uniform float c7 = 2.7555381529964506626129150390625e-4;

        uniform float result = x * c7 + c6;
        result = x * result + c5;
        result = x * result + c4;
        result = x * result + c3;
        result = x * result + c2;
        result = x * result + one;
        result = x * result + one;

        // Compute 2^k (should differ for uniform float and double, but I'll avoid
        // it for now and just do uniform floats)
        const uniform int fpbias = 127;
        uniform int biased_n = k + fpbias;
        uniform bool overflow = k_real > fpbias;
        // Minimum exponent is -126, so if k is <= -127 (k + 127 <= 0)
        // we've got underflow. -127 * ln(2) -> -88.02. So the most
        // negative uniform float input that doesn't result in zero is like -88.
        uniform bool underflow = k_real <= -fpbias;
        const uniform int InfBits = 0x7f800000;
        biased_n <<= 23;
        // Reuniform interpret this thing as uniform float
        uniform float two_to_the_n = floatbits(biased_n);
        // Handle both doubles and uniform floats (hopefully eliding the copy for uniform float)
        uniform float elemtype_2n = two_to_the_n;
        result *= elemtype_2n;
        result = overflow ? floatbits(InfBits) : result;
        result = underflow ? 0. : result;
        return result;
    }
}

__declspec(safe) static inline uniform float expm1(uniform float x) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_expm1f(x);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision:
        // - fixup range: <15 ULP
        // - other range: theoretically about ~2.5 times bigger than `exp`;
        //   <10 ULP in practice on reasonably-small input values (|x| < 87)
        //   with the current fast-math-lib `exp` implementation.
        // Support special values (e.g. NaN, Inf, subnormals) if `exp` support
        // it too, but `exp` currently does not support them.

        uniform float res = exp(x) - 1.0;

        // Sollya:
        // fpminimax(exp(x)-1, [|2,3,4,5|], [|single...|], [-0.35;0.35], 0+x);
        const uniform float c2 = +4.9999380110e-01;
        const uniform float c3 = +1.6666838527e-01;
        const uniform float c4 = +4.1876520960e-02;
        const uniform float c5 = +8.3380723370e-03;

        uniform float fixup = c5;
        fixup = x * fixup + c4;
        fixup = x * fixup + c3;
        fixup = x * fixup + c2;
        fixup = x * fixup + 1.0;
        fixup = x * fixup;

        return select(abs(x) < 0.35, fixup, res);
    } else if (__math_lib == __math_lib_ispc) {

        // Precision:
        // - fixup range: <5 ULP
        // - other range: theoretically about ~1.5 times bigger than `exp`;
        //   <5 ULP in practice with the current `exp` implementation.

        uniform float res = exp(x) - 1.0;

        // Sollya:
        // fpminimax(exp(x)-1, [|2,3,4,5,6|], [|single...|], [-0.5;0.5], 0+x);
        const uniform float c2 = +4.9999982120e-01;
        const uniform float c3 = +1.6666135190e-01;
        const uniform float c4 = +4.1669137780e-02;
        const uniform float c5 = +8.4023978560e-03;
        const uniform float c6 = +1.3871003175e-03;

        uniform float fixup = c6;
        fixup = x * fixup + c5;
        fixup = x * fixup + c4;
        fixup = x * fixup + c3;
        fixup = x * fixup + c2;
        fixup = x * fixup + 1.0;
        fixup = x * fixup;

        return select(abs(x) < 0.5, fixup, res);
    }
}

__declspec(safe) static inline float expm1(float x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_expm1f(x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_expm1f(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        float res = exp(x) - 1.0;

        const float c2 = +4.9999380110e-01;
        const float c3 = +1.6666838527e-01;
        const float c4 = +4.1876520960e-02;
        const float c5 = +8.3380723370e-03;

        float fixup = c5;
        fixup = x * fixup + c4;
        fixup = x * fixup + c3;
        fixup = x * fixup + c2;
        fixup = x * fixup + 1.0;
        fixup = x * fixup;

        return select(abs(x) < 0.35, fixup, res);
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        float res = exp(x) - 1.0;

        const float c2 = +4.9999982120e-01;
        const float c3 = +1.6666135190e-01;
        const float c4 = +4.1669137780e-02;
        const float c5 = +8.4023978560e-03;
        const float c6 = +1.3871003175e-03;

        float fixup = c6;
        fixup = x * fixup + c5;
        fixup = x * fixup + c4;
        fixup = x * fixup + c3;
        fixup = x * fixup + c2;
        fixup = x * fixup + 1.0;
        fixup = x * fixup;

        return select(abs(x) < 0.5, fixup, res);
    }
}

// Range reduction for logarithms takes log(x) -> log(2^n * y) -> n
// * log(2) + log(y) where y is the reduced range (usually in [1/2,1)).
// Warning: this function does not support subnormal numbers.
__declspec(safe) static inline void __range_reduce_log(float input, varying float *uniform reduced,
                                                       varying int *uniform exponent) {
    int int_version = intbits(input);
    // single precision = SEEE EEEE EMMM MMMM MMMM MMMM MMMM MMMM
    // exponent mask    = 0111 1111 1000 0000 0000 0000 0000 0000
    //                    0x7  0xF  0x8  0x0  0x0  0x0  0x0  0x0
    // non-exponent     = 1000 0000 0111 1111 1111 1111 1111 1111
    //                  = 0x8  0x0  0x7  0xF  0xF  0xF  0xF  0xF

    // const int exponent_mask(0x7F800000)
    static const int nonexponent_mask = 0x807FFFFF;

    // We want the reduced version to have an exponent of -1 which is -1 + 127 after biasing or 126
    static const int exponent_neg1 = (126l << 23);
    // NOTE(boulos): We don't need to mask anything out since we know
    // the sign bit has to be 0. If it's 1, we need to return infinity/nan
    // anyway (log(x), x = +-0 -> infinity, x < 0 -> NaN).
    int biased_exponent = int_version >> 23; // This number is [0, 255] but it means [-127, 128]

    int offset_exponent = biased_exponent + 1; // Treat the number as if it were 2^{e+1} * (1.m)/2
    *exponent = offset_exponent - 127;         // get the real value

    // Blend the offset_exponent with the original input (do this in
    // int for now, until I decide if float can have & and &not)
    int blended = (int_version & nonexponent_mask) | (exponent_neg1);
    *reduced = floatbits(blended);
}

__declspec(safe) static inline void __range_reduce_log(uniform float input, uniform float *uniform reduced,
                                                       uniform int *uniform exponent) {
    uniform int int_version = intbits(input);
    static const uniform int nonexponent_mask = 0x807FFFFF;

    static const uniform int exponent_neg1 = (126ul << 23);
    uniform int biased_exponent = int_version >> 23;
    uniform int offset_exponent = biased_exponent + 1;
    *exponent = offset_exponent - 127;

    uniform int blended = (int_version & nonexponent_mask) | (exponent_neg1);
    *reduced = floatbits(blended);
}

__declspec(safe) static inline uniform float log(uniform float x_full) {
    if (__have_native_transcendentals) {
        return __log_uniform_float(x_full);
    } else if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_logf(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <30 ULP
        // Does not support any special values:
        // zero and negative input values, subnormals numbers, inf and NaNs.
        // The the default version for implementation details.

        const uniform uint32 x_repr = intbits(x_full);

        const uniform float unstable_range_size = 0.2841;
        const uniform uint32 start_repr = intbits(1.0);
        const uniform uint32 end_repr = (uniform uint32)intbits(1.0 + unstable_range_size) + 1ul;
        const uniform uint32 unstable_range_ulp_size = end_repr - start_repr;
        const uniform bool close_to_one = x_repr - start_repr < unstable_range_ulp_size;

        uniform float reduced;
        uniform int exponent;

        __range_reduce_log(x_full, &reduced, &exponent);
        reduced = select(close_to_one, x_full, reduced);

        const uniform float ln2 = 6.931471825e-01;
        const uniform float scale = select(close_to_one, 0.0, (uniform float)exponent * ln2);

        // Initial Sollya version (then manually improved from it):
        // fpminimax(log(x+1), [|1,...,8|], [|single...|], [-0.5,0.2841]);
        const uniform float c1 = +1.0000009537e+00;
        const uniform float c2 = -4.9997970460e-01;
        const uniform float c3 = +3.3307218550e-01;
        const uniform float c4 = -2.5177517530e-01;
        const uniform float c5 = +2.0792385936e-01;
        const uniform float c6 = -1.2708663940e-01;
        const uniform float c7 = +9.3554288150e-02;
        const uniform float c8 = -4.0092557670e-01;

        const uniform float x = reduced - 1.0;
        const uniform float x2 = x * x + 1e-30;

        uniform float result = x * c8 + c7;
        result = x2 * result + (x * c6 + c5);
        result = x2 * result + (x * c4 + c3);
        result = x2 * result + (x * c2 + c1);
        result = x * result + scale;

        return result;
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <4 ULP

        const uniform uint32 x_repr = intbits(x_full);

        // Is a special input: +Inf, NaNs, negative numbers, subnormal ones
        const uniform bool special = x_repr + 0x80800000 < 0x81000000;

        if (any(special)) {

            // Rather-slow path

            const uniform float pinf = floatbits(0x7F800000);
            const uniform float ninf = floatbits(0xFF800000);
            const uniform float pnan = floatbits(0x7FC00000);

            if (isnan(x_full) || x_full == pinf)
                return x_full;

            if (x_full < 0.0)
                return pnan;

            if (x_full == 0.0)
                return ninf;

            // Subnormal values
            if (x_repr < 0x00800000) {
                x_full *= 1e36;

                uniform float reduced;
                uniform int exponent;

                __range_reduce_log(x_full, &reduced, &exponent);

                const uniform float ln1e36 = 8.2893063348e+01;
                const uniform float ln2 = 6.931471825e-01;
                const uniform float scale = (uniform float)exponent * ln2 - ln1e36;

                // Sollya:
                // fpminimax(log(x+1)/x, [|1,...,7|], [|single...|], [-0.5,-1e-8], 1);
                const uniform float c02 = -5.0002461670e-01;
                const uniform float c03 = +3.3212903140e-01;
                const uniform float c04 = -2.6966506240e-01;
                const uniform float c05 = +5.0812456760e-02;
                const uniform float c06 = -7.5164681670e-01;
                const uniform float c07 = -1.0362093450e+00;
                const uniform float c08 = -1.1606367826e+00;

                const uniform float x = reduced - 1.0;
                const uniform float x2 = x * x;

                // Estrin's scheme used to reduce latency
                uniform float result = x * c08 + c07;
                result = x2 * result + (x * c06 + c05);
                result = x2 * result + (x * c04 + c03);
                result = x2 * result + (x * c02 + 1.0);
                result = x * result + scale;

                return result;
            }
        }

        // Is in the range [1.0;1.0+unstable_range_size]
        const uniform float unstable_range_size = 0.285;
        const uniform uint32 start_repr = intbits(1.0);
        const uniform uint32 end_repr = (uniform uint32)intbits(1.0 + unstable_range_size) + 1ul;
        const uniform uint32 unstable_range_ulp_size = end_repr - start_repr;
        const uniform bool close_to_one = x_repr - start_repr < unstable_range_ulp_size;

        uniform float reduced;
        uniform int exponent;

        __range_reduce_log(x_full, &reduced, &exponent);
        reduced = select(close_to_one, x_full, reduced);

        // The extended range is meant to avoid a catastrophic
        // cancellation happenning with the usual approach.
        const uniform float ln2 = 6.931471825e-01;
        const uniform float scale = select(close_to_one, 0.0, (uniform float)exponent * ln2);

        // Initial Sollya version:
        // fpminimax(log(x+1), [|2,...,10|], [|single...|], [-0.5,0.285], x);
        // The final version is a manual refinement based on it.
        const uniform float c02 = -4.9999991060e-01;
        const uniform float c03 = +3.3335432410e-01;
        const uniform float c04 = -2.4996809660e-01;
        const uniform float c05 = +1.9873061776e-01;
        const uniform float c06 = -1.6905537250e-01;
        const uniform float c07 = +1.6525325180e-01;
        const uniform float c08 = -7.3399633170e-02;
        const uniform float c09 = -4.0101176130e-03;
        const uniform float c10 = -4.4349682330e-01;

        // Adding a tiny epsilon reduces the generation of subnormal numbers.
        // This massively improves the execution time when 1e-23 < x < 1e-19 on
        // CPUs computing subnormal numbers much slower (e.g. most Intel CPUs).
        const uniform float x = reduced - 1.0;
        const uniform float x2 = x * x + 1e-30;

        // Estrin's scheme used to reduce latency
        uniform float result = x * c10 + c09;
        result = x2 * result + (x * c08 + c07);
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;

        return result;
    }
}

__declspec(safe) static inline float log(float x_full) {
    if (__have_native_transcendentals) {
        return __log_varying_float(x_full);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_logf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_logf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        const uint32 x_repr = intbits(x_full);

        const float unstable_range_size = 0.2841;
        const uint32 start_repr = intbits(1.0);
        const uint32 end_repr = (uint32)intbits(1.0 + unstable_range_size) + 1ul;
        const uint32 unstable_range_ulp_size = end_repr - start_repr;
        const bool close_to_one = x_repr - start_repr < unstable_range_ulp_size;

        float reduced;
        int exponent;

        __range_reduce_log(x_full, &reduced, &exponent);
        reduced = select(close_to_one, x_full, reduced);

        const float ln2 = 6.931471825e-01;
        const float scale = select(close_to_one, 0.0, (float)exponent * ln2);

        const float c1 = +1.0000009537e+00;
        const float c2 = -4.9997970460e-01;
        const float c3 = +3.3307218550e-01;
        const float c4 = -2.5177517530e-01;
        const float c5 = +2.0792385936e-01;
        const float c6 = -1.2708663940e-01;
        const float c7 = +9.3554288150e-02;
        const float c8 = -4.0092557670e-01;

        const float x = reduced - 1.0;
        const float x2 = x * x + 1e-30;

        float result = x * c8 + c7;
        result = x2 * result + (x * c6 + c5);
        result = x2 * result + (x * c4 + c3);
        result = x2 * result + (x * c2 + c1);
        result = x * result + scale;

        return result;
    } else if (__math_lib == __math_lib_ispc) {
        const uint32 x_repr = intbits(x_full);
        const bool special = x_repr + 0x80800000 < 0x81000000;

        if (any(special)) {
            const float pinf = floatbits(0x7F800000);
            const float ninf = floatbits(0xFF800000);
            const float pnan = floatbits(0x7FC00000);

            if (isnan(x_full) || x_full == pinf)
                return x_full;

            if (x_full < 0.0)
                return pnan;

            if (x_full == 0.0)
                return ninf;

            if (x_repr < 0x00800000) {
                x_full *= 1e36;

                float reduced;
                int exponent;

                __range_reduce_log(x_full, &reduced, &exponent);

                const float ln1e36 = 8.2893063348e+01;
                const float ln2 = 6.931471825e-01;
                const float scale = (float)exponent * ln2 - ln1e36;

                const float c02 = -5.0002461670e-01;
                const float c03 = +3.3212903140e-01;
                const float c04 = -2.6966506240e-01;
                const float c05 = +5.0812456760e-02;
                const float c06 = -7.5164681670e-01;
                const float c07 = -1.0362093450e+00;
                const float c08 = -1.1606367826e+00;

                const float x = reduced - 1.0;
                const float x2 = x * x;

                float result = x * c08 + c07;
                result = x2 * result + (x * c06 + c05);
                result = x2 * result + (x * c04 + c03);
                result = x2 * result + (x * c02 + 1.0);
                result = x * result + scale;

                return result;
            }
        }

        const float unstable_range_size = 0.285;
        const uint32 start_repr = intbits(1.0);
        const uint32 end_repr = (uint32)intbits(1.0 + unstable_range_size) + 1ul;
        const uint32 unstable_range_ulp_size = end_repr - start_repr;
        const bool close_to_one = x_repr - start_repr < unstable_range_ulp_size;

        float reduced;
        int exponent;

        __range_reduce_log(x_full, &reduced, &exponent);
        reduced = select(close_to_one, x_full, reduced);

        const float ln2 = 6.931471825e-01;
        const float scale = select(close_to_one, 0.0, (float)exponent * ln2);

        const float c02 = -4.9999991060e-01;
        const float c03 = +3.3335432410e-01;
        const float c04 = -2.4996809660e-01;
        const float c05 = +1.9873061776e-01;
        const float c06 = -1.6905537250e-01;
        const float c07 = +1.6525325180e-01;
        const float c08 = -7.3399633170e-02;
        const float c09 = -4.0101176130e-03;
        const float c10 = -4.4349682330e-01;

        const float x = reduced - 1.0;
        const float x2 = x * x + 1e-30;

        float result = x * c10 + c09;
        result = x2 * result + (x * c08 + c07);
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;

        return result;
    }
}

__declspec(safe) static inline uniform float log1p(uniform float x_full) {
    const uniform float ln2 = 6.931471825e-01;
    const uniform float pinf = floatbits(0x7F800000);
    const uniform float ninf = floatbits(0xFF800000);
    const uniform float pnan = floatbits(0x7FC00000);

    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_log1pf(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <30 ULP
        // Does not support the following special input values:
        // NaNs, numbers <= -1.0, +inf.
        // Subnormal numbers are correctly supported.
        // This implementation treat -0.0 as +0.0.

        // Initial Sollya version (then manually improved from it):
        // range = [-0.5;0.285];
        // P = fpminimax(log(x+1), [|2,...,10|], [|single...|], range, x);
        const uniform float c02 = -4.9998131400e-01;
        const uniform float c03 = +3.3315047620e-01;
        const uniform float c04 = -2.5166967510e-01;
        const uniform float c05 = +2.0652699470e-01;
        const uniform float c06 = -1.2913592160e-01;
        const uniform float c07 = +1.0059609264e-01;
        const uniform float c08 = -3.8902461530e-01;

        uniform float reduced;
        uniform int exponent;

        const uniform bool close_to_zero = abs(x_full) < 0.285;

        __range_reduce_log(1.0 + x_full, &reduced, &exponent);

        const uniform float x = select(close_to_zero, x_full, reduced - 1.0);
        const uniform float x2 = x * x + 1e-30;
        uniform float scale = (uniform float)exponent * ln2;
        scale = select(close_to_zero, 0.0, scale);

        uniform float result = x * c08 + c07;
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;
        return result;
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <5 ULP

        // Initial Sollya version (then manually improved from it):
        // range = [-0.5;0.295];
        // P = fpminimax(log(x+1), [|2,...,10|], [|single...|], range, x);
        const uniform float c02 = -4.9999958280e-01;
        const uniform float c03 = +3.3335196970e-01;
        const uniform float c04 = -2.4999302626e-01;
        const uniform float c05 = +1.9877377150e-01;
        const uniform float c06 = -1.6865649820e-01;
        const uniform float c07 = +1.6505996883e-01;
        const uniform float c08 = -7.5250349940e-02;
        const uniform float c09 = -2.9322218620e-03;
        const uniform float c10 = -4.3928396700e-01;

        uniform float reduced;
        uniform int exponent;

        const uniform bool close_to_zero = abs(x_full) < 0.295;

        __range_reduce_log(1.0 + x_full, &reduced, &exponent);

        const uniform float x = select(close_to_zero, x_full, reduced - 1.0);
        const uniform float x2 = x * x + 1e-30;

        // The -0.0 is only meant to compute -0.0 input values correctly
        uniform float scale = (uniform float)exponent * ln2;
        scale = select(close_to_zero, -0.0, scale);

        uniform float result = x * c10 + c09;
        result = x2 * result + (x * c08 + c07);
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;

        // Correct special values
        const uniform float tmp = select(x_full == -1.0, ninf, pnan);
        result = select(x_full > -1.0, result, tmp);
        result = select(x_full == pinf, pinf, result);
        return result;
    }
}

__declspec(safe) static inline float log1p(float x_full) {
    const float ln2 = 6.931471825e-01;
    const float pinf = floatbits(0x7F800000);
    const float ninf = floatbits(0xFF800000);
    const float pnan = floatbits(0x7FC00000);

    if (__math_lib == __math_lib_svml) {
        return __svml_log1pf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_log1pf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        const float c02 = -4.9998131400e-01;
        const float c03 = +3.3315047620e-01;
        const float c04 = -2.5166967510e-01;
        const float c05 = +2.0652699470e-01;
        const float c06 = -1.2913592160e-01;
        const float c07 = +1.0059609264e-01;
        const float c08 = -3.8902461530e-01;

        float reduced;
        int exponent;

        const bool close_to_zero = abs(x_full) < 0.285;

        __range_reduce_log(1.0 + x_full, &reduced, &exponent);

        const float x = select(close_to_zero, x_full, reduced - 1.0);
        const float x2 = x * x + 1e-30;
        float scale = (float)exponent * ln2;
        scale = select(close_to_zero, 0.0, scale);

        float result = x * c08 + c07;
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;
        return result;
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        const float c02 = -4.9999958280e-01;
        const float c03 = +3.3335196970e-01;
        const float c04 = -2.4999302626e-01;
        const float c05 = +1.9877377150e-01;
        const float c06 = -1.6865649820e-01;
        const float c07 = +1.6505996883e-01;
        const float c08 = -7.5250349940e-02;
        const float c09 = -2.9322218620e-03;
        const float c10 = -4.3928396700e-01;

        float reduced;
        int exponent;

        const bool close_to_zero = abs(x_full) < 0.295;

        __range_reduce_log(1.0 + x_full, &reduced, &exponent);

        const float x = select(close_to_zero, x_full, reduced - 1.0);
        const float x2 = x * x + 1e-30;

        float scale = (float)exponent * ln2;
        scale = select(close_to_zero, -0.0, scale);

        float result = x * c10 + c09;
        result = x2 * result + (x * c08 + c07);
        result = x2 * result + (x * c06 + c05);
        result = x2 * result + (x * c04 + c03);
        result = x2 * result + (x * c02 + 1.0);
        result = x * result + scale;

        const float tmp = select(x_full == -1.0, ninf, pnan);
        result = select(x_full > -1.0, result, tmp);
        result = select(x_full == pinf, pinf, result);
        return result;
    }
}

__declspec(safe) static inline float pow(float a, float b) {
    if (__have_native_transcendentals) {
        return __pow_varying_float(a, b);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_powf(a, b);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_powf(extract(a, i), extract(b, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        return exp(b * log(a));
    }
}

__declspec(safe) static inline uniform float pow(uniform float a, uniform float b) {
    if (__have_native_transcendentals) {
        return __pow_uniform_float(a, b);
    }
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_powf(a, b);
    } else if (__math_lib == __math_lib_ispc || __math_lib == __math_lib_ispc_fast) {
        return exp(b * log(a));
    }
}

__declspec(safe) static inline uniform uint32 _fast_div3(uniform uint32 x) {
    return ((uniform uint64)x * (uniform uint64)1431655766) >> (uniform uint64)32;
}

__declspec(safe) static inline varying uint32 _fast_div3(varying uint32 x) {
    return ((varying uint64)x * (varying uint64)1431655766) >> (varying uint64)32;
}

// Cube root approximation using bit hacks for 32-bit float adapted from Kahan's cbrt.
// See "Computing a Real Cube Root" by W. Kahan (https://csclub.uwaterloo.ca/~pbarfuss/qbrt.pdf).
// Improvement of the algorithm provided in https://web.archive.org/20131227144655/http://metamerist.com/cbrt/cbrt.htm.
// The template is only meant to support both uniform and varying input variables in a single function.
// It only supports floats.
template <typename T> __declspec(safe) static inline T _cbrtf_base(T x) {
    // Note: using memcpy is strangelly a bit faster, but also unsafe
    return floatbits(_fast_div3(intbits(x)) + 709921077u);
}

// Compute a Newton-Raphson iteration (applied to the cube root function).
// Support both simple and double precision as well as both uniform and varying types.
// This method has a quadratic convergance rate (only close to the target value).
template <typename T> __declspec(safe) static inline T _cbrt_newton_iter(T a, T b) {
    const uniform T third = (uniform T)1 / (uniform T)3;
    return a - third * (a - b / (a * a));
}

// Compute an iteration of the Halley's method (applied to the cube root function).
// Support both simple and double precision as well as both uniform and varying types.
// See "Machine Method for the Extraction of Cube Root" by Otis E. Lancaster (https://www.jstor.org/stable/2279437).
// This method has a cubic convergance rate (only close to the target value).
template <typename T> __declspec(safe) static inline T _cbrt_halley_iter(T a, T b) {
    // Note: the order of the operation matters since numbers can quickly overflow
    // while computing the cubic root of really huge/tiny numbers.
    const uniform T two = (uniform T)2;
    const T a3 = a * a * a;
    const T correction_factor = (a3 + two * b) / (two * a3 + b);
    return a * correction_factor;
}

// Compute the cubic root of a given number.
// The template is only meant to support both uniform and varying input variables in a single function.
// It only supports floats.
// Support special values like NaN, Inf, -Inf, -0.0 and subnormal numbers.
// Precision for all numbers:
//  - if precise is true:   <5 ULP
//  - if precise is false: <30 ULP
template <typename T> __declspec(safe) static inline T _cbrtf(T x, uniform bool precise) {
    const T x_abs = abs(x);
    T res;

    // Correct the input value so it is neither too big nor too small
    const T xFactor = select(x_abs < 1e-12, 1e18, select(x_abs > 1e12, 1e-18, (uniform T)1));
    const T x_corr = x_abs * xFactor;

    // Compute an initial guess value and iteratively improve it.
    // This methods converge very quickly if the initial guess is good.
    // This approach only actually work well only if the numbers are neither
    // very small or very big ones.
    res = _cbrtf_base(x_corr);

    if (precise) {
        res = _cbrt_halley_iter(res, x_corr);
        res = _cbrt_halley_iter(res, x_corr);
    } else {
        res = _cbrt_newton_iter(res, x_corr);
        res = _cbrt_newton_iter(res, x_corr);
    }

    // Correct the result based on the (above) input correction
    const T resFactor = select(x_abs < 1e-12, 1e-6, select(x_abs > 1e12, 1e6, (uniform T)1));
    res *= resFactor;

    // Transfer the sign
    const T signed_res = floatbits(intbits(res) | signbits(x));

    // Special values are: NaN, +Inf, -Inf, -0, +0
    return select((intbits(x) & 0x7F800000) == 0x7F800000 || x == 0.0f, x, signed_res);
}

__declspec(safe) static inline uniform float cbrt(uniform float x) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_cbrtf(x);
    } else if (__math_lib == __math_lib_ispc_fast) {
        return _cbrtf(x, false);
    } else if (__math_lib == __math_lib_ispc) {
        return _cbrtf(x, true);
    }
}

__declspec(safe) static inline float cbrt(float x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_cbrtf(x);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_cbrtf(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {
        return _cbrtf(x, false);
    } else if (__math_lib == __math_lib_ispc) {
        return _cbrtf(x, true);
    }
}

__declspec(safe) static inline uniform float erf(uniform float x_full) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_erff(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <80 ULP for all numbers (including subnormals).
        // Correctly supports infinities and NaNs (though not required).

        uniform float x = abs(x_full);

        // Sollya:
        // fpminimax(erf(x), [|1,...,9|], [|single...|], [0.0;2.05]);
        const uniform float a1 = +1.1283835173e+0;
        const uniform float a2 = -2.8501544150e-4;
        const uniform float a3 = -3.7310788040e-1;
        const uniform float a4 = -1.1739640497e-2;
        const uniform float a5 = +1.3303731380e-1;
        const uniform float a6 = -1.2685810216e-2;
        const uniform float a7 = -3.3539909870e-2;
        const uniform float a8 = +1.4499780722e-2;
        const uniform float a9 = -1.8588694510e-3;

        uniform float a_res = a9;
        a_res = x * a_res + a8;
        a_res = x * a_res + a7;
        a_res = x * a_res + a6;
        a_res = x * a_res + a5;
        a_res = x * a_res + a4;
        a_res = x * a_res + a3;
        a_res = x * a_res + a2;
        a_res = x * a_res + a1;
        a_res = x * a_res;

        // Sollya:
        // fpminimax(erf(x), [|1,...,7|], [|single...|], [2.05;3.30]);
        const uniform float b1 = +1.4865404367e+0;
        const uniform float b2 = -6.6944038870e-1;
        const uniform float b3 = -7.4785791340e-2;
        const uniform float b4 = +1.7385362090e-1;
        const uniform float b5 = -6.4946919680e-2;
        const uniform float b6 = +1.0646330193e-2;
        const uniform float b7 = -6.7520828450e-4;

        uniform float b_res = b7;
        b_res = x * b_res + b6;
        b_res = x * b_res + b5;
        b_res = x * b_res + b4;
        b_res = x * b_res + b3;
        b_res = x * b_res + b2;
        b_res = x * b_res + b1;
        b_res = x * b_res;

        uniform float res = select(x > 3.30, 1.0, select(x < 2.05, a_res, b_res));
        uniform float signed_res = floatbits(intbits(res) | signbits(x_full));
        return signed_res;
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <3 ULP

        // Implement a tweaked approximation of Abramowitz and Stegun (AS)
        // for x > 1.261, combined with a polynomial approx in [0.0;1.261].
        // Indeed, the absolute error of the initial AS approx is 3e-7
        // for x > 0, but the relative error is higher close to 0.
        // The relative error of the tweaked AS approx is <3 ULP.
        // The relative error of polynomial approx is <3 ULP.
        // The Horner's sheme is used mainly for better performance.
        // The AS coefficients have been optimized for the target range.
        // See https://en.wikipedia.org/wiki/Error_function

        // Note: a constant >= 4.0 is needed here so erf(inf) is exactly 1.0.
        uniform float x_abs = abs(x_full);
        uniform float x = select(x_abs > 4.0, 4.0, x_abs);

        // Basic polynomial approx (for values close to zero)

        // Initial Sollya version (then manually improved from it):
        // fpminimax(erf(x), [|1,3,5,7,9,11,13|], [|single...|], [0.0;1.261]);
        const uniform float p0 = +1.1283791065e+00;
        const uniform float p1 = -3.7612408400e-01;
        const uniform float p2 = +1.1281772703e-01;
        const uniform float p3 = -2.6794515550e-02;
        const uniform float p4 = +5.0977407950e-03;
        const uniform float p5 = -7.3578109730e-04;
        const uniform float p6 = +6.0578728150e-05;

        uniform float x2 = x * x + 1e-30;
        uniform float p_res = p6;
        p_res = x2 * p_res + p5;
        p_res = x2 * p_res + p4;
        p_res = x2 * p_res + p3;
        p_res = x2 * p_res + p2;
        p_res = x2 * p_res + p1;
        p_res = x2 * p_res + p0;
        p_res = x * p_res;

        // Tweaked Abramowitz and Stegun approximation

        // Initial Sollya version (then manually improved from it):
        // f = 1/(1-erf(x))^(1/16);
        // fpminimax(f, [|0,1,2,3,4,5|], [|single...|], [1.258;2.18]);
        const uniform float a0 = +9.9798190600e-01;
        const uniform float a1 = +7.7553816140e-02;
        const uniform float a2 = +3.2319165770e-02;
        const uniform float a3 = +1.6566971320e-02;
        const uniform float a4 = -2.7288584970e-03;
        const uniform float a5 = +8.4786931980e-04;

        uniform float a_res = a5;
        a_res = x2 * a_res + (x * a4 + a3);
        a_res = x2 * a_res + (x * a2 + a1);
        a_res = x * a_res + a0;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = 1.0 - 1.0 / a_res;

        // Selection by range (with implicit handling of NaN)
        uniform float res = select(x > 1.261, a_res, p_res);

        // Transfer the sign
        uniform float signed_res = floatbits(intbits(res) | signbits(x_full));

        return signed_res;
    }
}

__declspec(safe) static inline float erf(float x_full) {
    if (__math_lib == __math_lib_svml) {
        return __svml_erff(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_erff(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        float x = abs(x_full);

        const float a1 = +1.1283835173e+0;
        const float a2 = -2.8501544150e-4;
        const float a3 = -3.7310788040e-1;
        const float a4 = -1.1739640497e-2;
        const float a5 = +1.3303731380e-1;
        const float a6 = -1.2685810216e-2;
        const float a7 = -3.3539909870e-2;
        const float a8 = +1.4499780722e-2;
        const float a9 = -1.8588694510e-3;

        float a_res = a9;
        a_res = x * a_res + a8;
        a_res = x * a_res + a7;
        a_res = x * a_res + a6;
        a_res = x * a_res + a5;
        a_res = x * a_res + a4;
        a_res = x * a_res + a3;
        a_res = x * a_res + a2;
        a_res = x * a_res + a1;
        a_res = x * a_res;

        const float b1 = +1.4865404367e+0;
        const float b2 = -6.6944038870e-1;
        const float b3 = -7.4785791340e-2;
        const float b4 = +1.7385362090e-1;
        const float b5 = -6.4946919680e-2;
        const float b6 = +1.0646330193e-2;
        const float b7 = -6.7520828450e-4;

        float b_res = b7;
        b_res = x * b_res + b6;
        b_res = x * b_res + b5;
        b_res = x * b_res + b4;
        b_res = x * b_res + b3;
        b_res = x * b_res + b2;
        b_res = x * b_res + b1;
        b_res = x * b_res;

        float res = select(x > 3.30, 1.0, select(x < 2.05, a_res, b_res));
        float signed_res = floatbits(intbits(res) | signbits(x_full));
        return signed_res;
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        float x_abs = abs(x_full);
        float x = select(x_abs > 4.0, 4.0, x_abs);

        const float p0 = +1.1283791065e+00;
        const float p1 = -3.7612408400e-01;
        const float p2 = +1.1281772703e-01;
        const float p3 = -2.6794515550e-02;
        const float p4 = +5.0977407950e-03;
        const float p5 = -7.3578109730e-04;
        const float p6 = +6.0578728150e-05;

        float x2 = x * x + 1e-30;
        float p_res = p6;
        p_res = x2 * p_res + p5;
        p_res = x2 * p_res + p4;
        p_res = x2 * p_res + p3;
        p_res = x2 * p_res + p2;
        p_res = x2 * p_res + p1;
        p_res = x2 * p_res + p0;
        p_res = x * p_res;

        const float a0 = +9.9798190600e-01;
        const float a1 = +7.7553816140e-02;
        const float a2 = +3.2319165770e-02;
        const float a3 = +1.6566971320e-02;
        const float a4 = -2.7288584970e-03;
        const float a5 = +8.4786931980e-04;

        float a_res = a5;
        a_res = x2 * a_res + (x * a4 + a3);
        a_res = x2 * a_res + (x * a2 + a1);
        a_res = x * a_res + a0;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = a_res * a_res;
        a_res = 1.0 - 1.0 / a_res;

        float res = select(x > 1.261, a_res, p_res);
        float signed_res = floatbits(intbits(res) | signbits(x_full));
        return signed_res;
    }
}

// Mixed-precision exp designed mainly for `erfc`.
// It only supports negative `x_full` values.
// More precise than the single-precision exp:
// It avoid a numerical instability for big negative input single-precision
// values. Indeed, without it, the error of `exp(-x*x)` for x in [0;10] can
// reach up to 70 ULP! With this function, it is <3.5 ULP.
__declspec(safe) static inline uniform float _erfc_neg_exp(uniform double x_full) {
    const uniform double ln2 = 6.9314718055994531e-1d;
    const uniform double one_over_ln2 = 1.4426950216293335d;

    uniform double scaled = x_full * one_over_ln2;
    uniform double k_real = floor(scaled);
    uniform int64 k = (uniform int64)k_real;

    uniform float x = (uniform float)(x_full - k_real * ln2);

    const uniform float one = 1.0;
    const uniform float c2 = 4.9999991059e-1;
    const uniform float c3 = 1.6666841507e-1;
    const uniform float c4 = 4.1653905064e-2;
    const uniform float c5 = 8.3788307384e-3;
    const uniform float c6 = 1.3043793151e-3;
    const uniform float c7 = 2.7555381530e-4;

    uniform float poly = c7;
    poly = x * poly + c6;
    poly = x * poly + c5;
    poly = x * poly + c4;
    poly = x * poly + c3;
    poly = x * poly + c2;
    poly = x * poly + one;
    poly = x * poly + one;

    const uniform int64 fpbias = 1023;
    uniform int64 biased_n = k + fpbias;
    uniform bool underflow = k_real <= -fpbias;
    biased_n <<= 52;
    uniform float two_to_the_n = (uniform float)doublebits(biased_n);
    uniform float result = poly * two_to_the_n;
    result = underflow ? 0.0 : result;
    return result;
}

// See the uniform version for more information
__declspec(safe) static inline float _erfc_neg_exp(double x_full) {
    const double ln2 = 6.9314718055994531e-1d;
    const double one_over_ln2 = 1.4426950216293335d;

    double scaled = x_full * one_over_ln2;
    double k_real = floor(scaled);
    int64 k = (int64)k_real;

    float x = (float)(x_full - k_real * ln2);

    const float one = 1.0;
    const float c2 = 4.9999991059e-1;
    const float c3 = 1.6666841507e-1;
    const float c4 = 4.1653905064e-2;
    const float c5 = 8.3788307384e-3;
    const float c6 = 1.3043793151e-3;
    const float c7 = 2.7555381530e-4;

    float poly = c7;
    poly = x * poly + c6;
    poly = x * poly + c5;
    poly = x * poly + c4;
    poly = x * poly + c3;
    poly = x * poly + c2;
    poly = x * poly + one;
    poly = x * poly + one;

    const int64 fpbias = 1023;
    int64 biased_n = k + fpbias;
    bool underflow = k_real <= -fpbias;
    biased_n <<= 52;
    float two_to_the_n = (float)doublebits(biased_n);
    float result = poly * two_to_the_n;
    result = underflow ? 0.0 : result;
    return result;
}

__declspec(safe) static inline uniform float erfc(uniform float x_full) {
    if (__math_lib == __math_lib_system || __math_lib == __math_lib_svml) {
        return __stdlib_erfcf(x_full);
    } else if (__math_lib == __math_lib_ispc_fast) {

        // Precision: <80 ULP for all numbers (including subnormals).
        // Correctly supports infinities and NaNs (though not required).

        uniform float x_sp = abs(x_full);
        x_sp = select(x_sp > 10.1, 10.1, x_sp);
        uniform double x_dp = (uniform double)x_sp;
        uniform double x2_dp = x_dp * x_dp;
        uniform double x4_dp = x2_dp * x2_dp;

        // Sollya:
        // fpminimax(log(erfc(x)), 12, [|double...|], [0.0;9.85], absolute);
        const uniform double c00 = +4.197320213670971199e-06d;
        const uniform double c01 = -1.128550949837619388e+00d;
        const uniform double c02 = -6.354525758129699886e-01d;
        const uniform double c03 = -1.059060072406146635e-01d;
        const uniform double c04 = +2.352663581509139776e-02d;
        const uniform double c05 = -3.458848280493454896e-03d;
        const uniform double c06 = +1.979893961261454264e-04d;
        const uniform double c07 = +4.052279761164515038e-05d;
        const uniform double c08 = -1.188755454524737013e-05d;
        const uniform double c09 = +1.484807869494198427e-06d;
        const uniform double c10 = -1.053828127691732596e-07d;
        const uniform double c11 = +4.124237406063885519e-09d;
        const uniform double c12 = -6.939009269178366423e-11d;

        // Estrin's scheme used to reduce latency
        uniform double poly = c12;
        poly = poly * x4_dp + ((c11 * x_dp + c10) * x2_dp + (c09 * x_dp + c08));
        poly = poly * x4_dp + ((c07 * x_dp + c06) * x2_dp + (c05 * x_dp + c04));
        poly = poly * x4_dp + ((c03 * x_dp + c02) * x2_dp + (c01 * x_dp + c00));

        uniform float res = _erfc_neg_exp(poly);
        return select(x_full < 0.0, 2.0 - res, res);
    } else if (__math_lib == __math_lib_ispc) {

        // Precision: <5 ULP

        uniform float x_sp = abs(x_full);
        x_sp = select(x_sp > 10.1, 10.1, x_sp);
        uniform double x_dp = (uniform double)x_sp;
        uniform double x2_dp = x_dp * x_dp;
        uniform double x4_dp = x2_dp * x2_dp;

        // Sollya:
        // fpminimax(log(erfc(x)), 16, [|double...|], [0.0;10.1], absolute);
        // Barely numerically stable but enough for single precision.
        const uniform double b00 = +7.528016144388386869e-08d;
        const uniform double b01 = -1.128383077173272797e+00d;
        const uniform double b02 = -6.365867488627735549e-01d;
        const uniform double b03 = -1.028749803222310105e-01d;
        const uniform double b04 = +1.926326185826594817e-02d;
        const uniform double b05 = +1.816228327158303892e-04d;
        const uniform double b06 = -1.854061368216941477e-03d;
        const uniform double b07 = +8.456966890087848701e-04d;
        const uniform double b08 = -2.394872983773455694e-04d;
        const uniform double b09 = +4.882621630801304224e-05d;
        const uniform double b10 = -7.427977311595383097e-06d;
        const uniform double b11 = +8.465890395524544962e-07d;
        const uniform double b12 = -7.133871899458714267e-08d;
        const uniform double b13 = +4.312533088891799362e-09d;
        const uniform double b14 = -1.768121176854640576e-10d;
        const uniform double b15 = +4.401051115149253867e-12d;
        const uniform double b16 = -5.018313772746684971e-14d;

        // Estrin's scheme used to reduce latency
        uniform double poly = b16;
        poly = poly * x4_dp + ((b15 * x_dp + b14) * x2_dp + (b13 * x_dp + b12));
        poly = poly * x4_dp + ((b11 * x_dp + b10) * x2_dp + (b09 * x_dp + b08));
        poly = poly * x4_dp + ((b07 * x_dp + b06) * x2_dp + (b05 * x_dp + b04));
        poly = poly * x4_dp + ((b03 * x_dp + b02) * x2_dp + (b01 * x_dp + b00));

        uniform float res = _erfc_neg_exp(poly);
        return select(x_full < 0.0, 2.0 - res, res);
    }
}

__declspec(safe) static inline float erfc(float x_full) {
    if (__math_lib == __math_lib_svml) {
        return __svml_erfcf(x_full);
    } else if (__math_lib == __math_lib_system) {
        float ret;
        foreach_active(i) {
            uniform float r = __stdlib_erfcf(extract(x_full, i));
            ret = insert(ret, i, r);
        }
        return ret;
    } else if (__math_lib == __math_lib_ispc_fast) {

        // See the uniform version for more information

        float x_sp = abs(x_full);
        x_sp = select(x_sp > 10.1, 10.1, x_sp);
        double x_dp = (double)x_sp;
        double x2_dp = x_dp * x_dp;
        double x4_dp = x2_dp * x2_dp;

        const double c00 = +4.197320213670971199e-06d;
        const double c01 = -1.128550949837619388e+00d;
        const double c02 = -6.354525758129699886e-01d;
        const double c03 = -1.059060072406146635e-01d;
        const double c04 = +2.352663581509139776e-02d;
        const double c05 = -3.458848280493454896e-03d;
        const double c06 = +1.979893961261454264e-04d;
        const double c07 = +4.052279761164515038e-05d;
        const double c08 = -1.188755454524737013e-05d;
        const double c09 = +1.484807869494198427e-06d;
        const double c10 = -1.053828127691732596e-07d;
        const double c11 = +4.124237406063885519e-09d;
        const double c12 = -6.939009269178366423e-11d;

        double poly = c12;
        poly = poly * x4_dp + ((c11 * x_dp + c10) * x2_dp + (c09 * x_dp + c08));
        poly = poly * x4_dp + ((c07 * x_dp + c06) * x2_dp + (c05 * x_dp + c04));
        poly = poly * x4_dp + ((c03 * x_dp + c02) * x2_dp + (c01 * x_dp + c00));

        float res = _erfc_neg_exp(poly);
        return select(x_full < 0.0, 2.0 - res, res);
    } else if (__math_lib == __math_lib_ispc) {

        // See the uniform version for more information

        float x_sp = abs(x_full);
        x_sp = select(x_sp > 10.1, 10.1, x_sp);
        double x_dp = (double)x_sp;
        double x2_dp = x_dp * x_dp;
        double x4_dp = x2_dp * x2_dp;

        const double b00 = +7.528016144388386869e-08d;
        const double b01 = -1.128383077173272797e+00d;
        const double b02 = -6.365867488627735549e-01d;
        const double b03 = -1.028749803222310105e-01d;
        const double b04 = +1.926326185826594817e-02d;
        const double b05 = +1.816228327158303892e-04d;
        const double b06 = -1.854061368216941477e-03d;
        const double b07 = +8.456966890087848701e-04d;
        const double b08 = -2.394872983773455694e-04d;
        const double b09 = +4.882621630801304224e-05d;
        const double b10 = -7.427977311595383097e-06d;
        const double b11 = +8.465890395524544962e-07d;
        const double b12 = -7.133871899458714267e-08d;
        const double b13 = +4.312533088891799362e-09d;
        const double b14 = -1.768121176854640576e-10d;
        const double b15 = +4.401051115149253867e-12d;
        const double b16 = -5.018313772746684971e-14d;

        double poly = b16;
        poly = poly * x4_dp + ((b15 * x_dp + b14) * x2_dp + (b13 * x_dp + b12));
        poly = poly * x4_dp + ((b11 * x_dp + b10) * x2_dp + (b09 * x_dp + b08));
        poly = poly * x4_dp + ((b07 * x_dp + b06) * x2_dp + (b05 * x_dp + b04));
        poly = poly * x4_dp + ((b03 * x_dp + b02) * x2_dp + (b01 * x_dp + b00));

        float res = _erfc_neg_exp(poly);
        return select(x_full < 0.0, 2.0 - res, res);
    }
}

///////////////////////////////////////////////////////////////////////////
// Transcendentals (16-bit float precision)

__declspec(safe) static inline float16 sqrt(float16 v) { return __sqrt_varying_half(v); }

__declspec(safe) static inline uniform float16 sqrt(uniform float16 v) { return __sqrt_uniform_half(v); }

__declspec(safe) static inline float16 rsqrt(float16 v) {
    if (__have_native_half_full_support) {
        return __rsqrt_varying_half(v);
    } else {
        return (float16)(rcp(sqrt((float)v)));
    }
}

__declspec(safe) static inline uniform float16 rsqrt(uniform float16 v) {
    if (__have_native_half_full_support) {
        return __rsqrt_uniform_half(v);
    } else {
        return (uniform float16)(rcp(sqrt((uniform float)v)));
    }
}

__declspec(safe) static inline float16 ldexp(float16 x, int n) {
    unsigned int16 ex = 0x7c00u;
    unsigned int16 ix = intbits(x);
    ex &= ix; // extract old exponent;
    float16 res = float16bits((unsigned int16)((ix & ~0x7c00u) | (((int16)n << 10) + ex)));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0f16) ? x : res;
    }
}

__declspec(safe) static inline uniform float16 ldexp(uniform float16 x, uniform int n) {
    uniform unsigned int16 ex = 0x7c00u;
    uniform unsigned int16 ix = intbits(x);
    ex &= ix; // extract old exponent;
    uniform float16 res = float16bits((uniform unsigned int16)((ix & ~0x7c00u) | (((int16)n << 10) + ex)));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0f16) ? x : res;
    }
}

__declspec(safe) static inline float16 frexp(float16 x, varying int *uniform pw2) {
    unsigned int16 ex = 0x7c00u; // exponent mask
    unsigned int16 ix = intbits(x);
    ex &= ix;
    ix &= ~0x7c00u;              // clear exponent
    *pw2 = (int)(ex >> 10) - 14; // compute exponent
    ix |= 0x3800u;               // insert exponent +1 in x
    return float16bits(ix);
}

__declspec(safe) static inline uniform float16 frexp(uniform float16 x, uniform int *uniform pw2) {
    uniform unsigned int16 ex = 0x7c00u; // exponent mask
    uniform unsigned int16 ix = intbits(x);
    ex &= ix;
    ix &= ~0x7c00u;                      // clear exponent
    *pw2 = (uniform int)(ex >> 10) - 14; // compute exponent
    ix |= 0x3800u;                       // insert exponent +1 in x
    return float16bits(ix);
}

// If no native trigonometry support, convert to float, get sin and convert to half back
__declspec(safe) static inline float16 sin(float16 x_full) {
    if (__have_native_trigonometry) {
        return __sin_varying_half(x_full);
    } else {
        return (float16)(sin((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 sin(uniform float16 x_full) {
    if (__have_native_trigonometry) {
        return __sin_uniform_half(x_full);
    } else {
        return (uniform float16)(sin((uniform float)x_full));
    }
}

__declspec(safe) static inline float16 sinh(float16 x_full) { return (float16)(sinh((float)x_full)); }

__declspec(safe) static inline uniform float16 sinh(uniform float16 x_full) {
    return (uniform float16)(sinh((uniform float)x_full));
}

__declspec(safe) static inline float16 asin(float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __asin_varying_half(x_full);
    } else {
        return (float16)(asin((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 asin(uniform float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __asin_uniform_half(x_full);
    } else {
        return (uniform float16)(asin((uniform float)x_full));
    }
}

__declspec(safe) static inline float16 cos(float16 x_full) {
    if (__have_native_trigonometry) {
        return __cos_varying_half(x_full);
    } else {
        return (float16)(cos((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 cos(uniform float16 x_full) {
    if (__have_native_trigonometry) {
        return __cos_uniform_half(x_full);
    } else {
        return (uniform float16)(cos((uniform float)x_full));
    }
}

__declspec(safe) static inline float16 cosh(float16 x_full) { return (float16)(cosh((float)x_full)); }

__declspec(safe) static inline uniform float16 cosh(uniform float16 x_full) {
    return (uniform float16)(cosh((uniform float)x_full));
}

__declspec(safe) static inline float16 tan(float16 x_full) {
    if (__have_native_trigonometry) {
        return __tan_varying_half(x_full);
    } else {
        return (float16)(tan((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 tan(uniform float16 x_full) {
    if (__have_native_trigonometry) {
        return __tan_uniform_half(x_full);
    } else {
        return (uniform float16)(tan((uniform float)x_full));
    }
}

__declspec(safe) static inline float16 tanh(float16 x_full) { return (float16)(tanh((float)x_full)); }

__declspec(safe) static inline uniform float16 tanh(uniform float16 x_full) {
    return (uniform float16)(tanh((uniform float)x_full));
}

__declspec(safe) static inline float16 acos(float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __acos_varying_half(x_full);
    } else {
        return (float16)(acos((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 acos(uniform float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __acos_uniform_half(x_full);
    } else {
        return (uniform float16)(acos((uniform float)x_full));
    }
}

__declspec(safe) static inline void sincos(float16 x_full, varying float16 *uniform sin_result,
                                           varying float16 *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_varying_half(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else {
        *sin_result = (float16)sin((float)x_full);
        *cos_result = (float16)cos((float)x_full);
    }
    return;
}

__declspec(safe) static inline void sincos(uniform float16 x_full, uniform float16 *uniform sin_result,
                                           uniform float16 *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_uniform_half(x_full, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else {
        *sin_result = (uniform float16)sin((uniform float)x_full);
        *cos_result = (uniform float16)cos((uniform float)x_full);
    }
    return;
}

__declspec(safe) static inline float16 atan(float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan_varying_half(x_full);
    } else {
        return (float16)(atan((float)x_full));
    }
}

__declspec(safe) static inline uniform float16 atan(uniform float16 x_full) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan_uniform_half(x_full);
    } else {
        return (uniform float16)(atan((uniform float)x_full));
    }
}

__declspec(safe) static inline float16 atan2(float16 y, float16 x) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan2_varying_half(y, x);
    } else {
        return (float16)(atan2((float)y, (float)x));
    }
}

__declspec(safe) static inline uniform float16 atan2(uniform float16 y, uniform float16 x) {
    if (__have_native_trigonometry && !__is_xe_target) {
        return __atan2_uniform_half(y, x);
    } else {
        return (uniform float16)(atan2((uniform float)y, (uniform float)x));
    }
}

__declspec(safe) static inline float16 exp(float16 x_full) { return __exp_varying_half(x_full); }

__declspec(safe) static inline uniform float16 exp(uniform float16 x_full) { return __exp_uniform_half(x_full); }

__declspec(safe) static inline float16 expm1(float16 x_full) { return (float16)(expm1((float)x_full)); }

__declspec(safe) static inline uniform float16 expm1(uniform float16 x_full) {
    return (uniform float16)(expm1((uniform float)x_full));
}

__declspec(safe) static inline float16 log(float16 x_full) { return __log_varying_half(x_full); }

__declspec(safe) static inline uniform float16 log(uniform float16 x_full) { return __log_uniform_half(x_full); }

__declspec(safe) static inline float16 log1p(float16 x_full) { return (float16)log1p((float)x_full); }

__declspec(safe) static inline uniform float16 log1p(uniform float16 x_full) {
    return (uniform float16)log1p((uniform float)x_full);
}

__declspec(safe) static inline float16 pow(float16 a, float16 b) { return __pow_varying_half(a, b); }

__declspec(safe) static inline uniform float16 pow(uniform float16 a, uniform float16 b) {
    return __pow_uniform_half(a, b);
}

__declspec(safe) static inline float16 erf(float16 x_full) { return (float16)(erf((float)x_full)); }

__declspec(safe) static inline uniform float16 erf(uniform float16 x_full) {
    return (uniform float16)(erf((uniform float)x_full));
}

__declspec(safe) static inline float16 erfc(float16 x_full) { return (float16)(erfc((float)x_full)); }

__declspec(safe) static inline uniform float16 erfc(uniform float16 x_full) {
    return (uniform float16)(erfc((uniform float)x_full));
}

///////////////////////////////////////////////////////////////////////////
// Transcendentals (double precision)

__declspec(safe) static inline double sqrt(double v) {
    if (__math_lib == __math_lib_svml) {
        return __svml_sqrtd(v);
    } else {
        return __sqrt_varying_double(v);
    }
}

__declspec(safe) static inline uniform double sqrt(uniform double v) { return __sqrt_uniform_double(v); }

#define RSQRTD(QUAL)                                                                                                   \
    __declspec(safe) static inline QUAL double __rsqrt_iterate_##QUAL##_double(QUAL double x, QUAL double y) {         \
        QUAL double xh = x * 0.5d;                                                                                     \
        y += y * (0.5d - xh * y * y);                                                                                  \
        y += y * (0.5d - xh * y * y);                                                                                  \
        return y;                                                                                                      \
    }                                                                                                                  \
    __declspec(safe) static inline QUAL double __rsqrt_safe_##QUAL##_double(QUAL double x) {                           \
        if (x <= 1.0e+33d && x >= 1.0e-33d)                                                                            \
            return __rsqrt_iterate_##QUAL##_double(x, rsqrt((QUAL float)x));                                           \
        QUAL int64 ex = intbits(x) & 0x7fe0000000000000;                                                               \
        QUAL double exp = doublebits(0x7fd0000000000000 - ex);         /* 1.0d/exponent  */                            \
        QUAL double exph = doublebits(0x5fe0000000000000 - (ex >> 1)); /* 1.0d/sqrt(exponent) */                       \
        QUAL double y = rsqrt((QUAL float)(x * exp));                                                                  \
        return __rsqrt_iterate_##QUAL##_double(x, y * exph);                                                           \
    }

RSQRTD(varying)
__declspec(safe) static inline double rsqrt(double v) {
    if (__math_lib == __math_lib_svml) {
        return __svml_invsqrtd(v);
    } else if (__have_native_rsqrtd) {
        return __rsqrt_varying_double(v);
    } else {
        return __rsqrt_safe_varying_double(v);
    }
}

RSQRTD(uniform)
__declspec(safe) static inline uniform double rsqrt(uniform double v) {
    if (__have_native_rsqrtd)
        return __rsqrt_uniform_double(v);
    else
        return __rsqrt_safe_uniform_double(v);
}

__declspec(safe) static inline double rsqrt_fast(double v) {
    if (__have_native_rsqrtd) {
        return __rsqrt_fast_varying_double(v);
    } else {
        return __rsqrt_safe_varying_double(v);
    }
}

__declspec(safe) static inline uniform double rsqrt_fast(uniform double v) {
    if (__have_native_rsqrtd) {
        return __rsqrt_fast_uniform_double(v);
    } else {
        return __rsqrt_safe_uniform_double(v);
    }
}

__declspec(safe) static inline double ldexp(double x, int n) {
    unsigned int64 ex = 0x7ff0000000000000;
    unsigned int64 ix = intbits(x);
    ex &= ix;
    double res = doublebits((varying unsigned int64)((ix & ~0x7ff0000000000000) | (((int64)n << 52) + ex)));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0d) ? x : res;
    }
}

__declspec(safe) static inline uniform double ldexp(uniform double x, uniform int n) {
    uniform unsigned int64 ex = 0x7ff0000000000000;
    uniform unsigned int64 ix = intbits(x);
    ex &= ix;
    uniform double res = doublebits((uniform unsigned int64)((ix & ~0x7ff0000000000000) | (((int64)n << 52) + ex)));
    if (__math_lib == __math_lib_ispc_fast) {
        return res;
    } else {
        // Return x if it is 0.0f. Otherwise clear and insert new exponent.
        return (x == 0.0d) ? x : res;
    }
}

__declspec(safe) static inline double frexp(double x, varying int *uniform pw2) {
    unsigned int64 ex = 0x7ff0000000000000; // exponent mask
    unsigned int64 ix = intbits(x);
    ex &= ix;
    ix &= ~0x7ff0000000000000;     // clear exponent
    *pw2 = (int)(ex >> 52) - 1022; // compute exponent
    ix |= 0x3fe0000000000000;      // insert exponent +1 in x
    return doublebits(ix);
}

__declspec(safe) static inline uniform double frexp(uniform double x, uniform int *uniform pw2) {
    uniform unsigned int64 ex = 0x7ff0000000000000; // exponent mask
    uniform unsigned int64 ix = intbits(x);
    ex &= ix;
    ix &= ~0x7ff0000000000000;     // clear exponent
    *pw2 = (int)(ex >> 52) - 1022; // compute exponent
    ix |= 0x3fe0000000000000;      // insert exponent +1 in x
    return doublebits(ix);
}

__declspec(safe) static inline double sin(double x) {
    if (__have_native_trigonometry) {
        return __sin_varying_double(x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_sind(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_sin(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double sin(uniform double x) {
    if (__have_native_trigonometry) {
        return __sin_uniform_double(x);
    } else
        return __stdlib_sin(x);
}

__declspec(safe) static inline double sinh(double x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_sinhd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_sinh(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double sinh(uniform double x) { return __stdlib_sinh(x); }

__declspec(safe) static inline uniform double asin(uniform double x) {
    if (__have_native_trigonometry) {
        return __asin_uniform_double(x);
    } else {
        return __stdlib_asin(x);
    }
}

__declspec(safe) static inline double asin(const double x) {
    if (__have_native_trigonometry) {
        return __asin_varying_double(x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_asind(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_asin(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline double cos(const double x) {
    if (__have_native_trigonometry) {
        return __cos_varying_double(x);
    }
    if (__math_lib == __math_lib_svml) {
        return __svml_cosd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_cos(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double cos(uniform double x) {
    if (__have_native_trigonometry) {
        return __cos_uniform_double(x);
    } else
        return __stdlib_cos(x);
}

__declspec(safe) static inline double cosh(double x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_coshd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_cosh(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double cosh(uniform double x) { return __stdlib_cosh(x); }

__declspec(safe) static inline double acos(const double v) {
    if (__have_native_trigonometry) {
        return __acos_varying_double(v);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_acosd(v);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_acos(extract(v, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double acos(const uniform double v) {
    if (__have_native_trigonometry)
        return __acos_uniform_double(v);
    else
        return __stdlib_acos(v);
}

__declspec(safe) static inline void sincos(double x, varying double *uniform sin_result,
                                           varying double *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_varying_double(x, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
        return;
    }
    if (__math_lib == __math_lib_svml) {
        __svml_sincosd(x, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else {
        foreach_active(i) {
            uniform double sr, cr;
            __stdlib_sincos(extract(x, i), (opaque_ptr_t)&sr, (opaque_ptr_t)&cr);
            *sin_result = insert(*sin_result, i, sr);
            *cos_result = insert(*cos_result, i, cr);
        }
    }
}

__declspec(safe) static inline void sincos(uniform double x, uniform double *uniform sin_result,
                                           uniform double *uniform cos_result) {
    if (__have_native_trigonometry) {
        __sincos_uniform_double(x, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
    } else
        __stdlib_sincos(x, (opaque_ptr_t)sin_result, (opaque_ptr_t)cos_result);
}

__declspec(safe) static inline double tan(double x) {
    if (__have_native_trigonometry) {
        return __tan_varying_double(x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_tand(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_tan(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double tan(uniform double x) {
    if (__have_native_trigonometry) {
        return __tan_uniform_double(x);
    } else
        return __stdlib_tan(x);
}

__declspec(safe) static inline double tanh(double x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_tanhd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_tanh(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double tanh(uniform double x) { return __stdlib_tanh(x); }

__declspec(safe) static inline double atan(double x) {
    if (__have_native_trigonometry) {
        return __atan_varying_double(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_atan(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double atan(uniform double x) {
    if (__have_native_trigonometry) {
        return __atan_uniform_double(x);
    } else
        return __stdlib_atan(x);
}

__declspec(safe) static inline double atan2(double y, double x) {
    if (__have_native_trigonometry) {
        return __atan2_varying_double(y, x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_atan2d(y, x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_atan2(extract(y, i), extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double atan2(uniform double y, uniform double x) {
    if (__have_native_trigonometry) {
        return __atan2_uniform_double(y, x);
    } else
        return __stdlib_atan2(y, x);
}

__declspec(safe) static inline double exp(double x) {
    if (__have_native_transcendentals) {
        return __exp_varying_double(x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_expd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_exp(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double exp(uniform double x) {
    if (__have_native_transcendentals) {
        return __exp_uniform_double(x);
    } else
        return __stdlib_exp(x);
}

__declspec(safe) static inline double expm1(double x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_expm1d(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_expm1(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double expm1(uniform double x) { return __stdlib_expm1(x); }

__declspec(safe) static inline double log(double x) {
    if (__have_native_transcendentals) {
        return __log_varying_double(x);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_logd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_log(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double log(uniform double x) {
    if (__have_native_transcendentals) {
        return __log_uniform_double(x);
    } else
        return __stdlib_log(x);
}

__declspec(safe) static inline double log1p(double x) {
    if (__math_lib == __math_lib_svml) {
        return __svml_log1pd(x);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_log1p(extract(x, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double log1p(uniform double x) { return __stdlib_log1p(x); }

__declspec(safe) static inline double pow(double a, double b) {
    if (__have_native_transcendentals) {
        return __pow_varying_double(a, b);
    } else if (__math_lib == __math_lib_svml) {
        return __svml_powd(a, b);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_pow(extract(a, i), extract(b, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double pow(uniform double a, uniform double b) {
    if (__have_native_transcendentals) {
        return __pow_uniform_double(a, b);
    } else
        return __stdlib_pow(a, b);
}

__declspec(safe) static inline double cbrt(double a) {
    if (__math_lib == __math_lib_svml) {
        return __svml_cbrtd(a);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_cbrt(extract(a, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double cbrt(uniform double a) { return __stdlib_cbrt(a); }

__declspec(safe) static inline double erf(double a) {
    if (__math_lib == __math_lib_svml) {
        return __svml_erfd(a);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_erf(extract(a, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double erf(uniform double a) { return __stdlib_erf(a); }

__declspec(safe) static inline double erfc(double a) {
    if (__math_lib == __math_lib_svml) {
        return __svml_erfcd(a);
    } else {
        double ret;
        foreach_active(i) {
            uniform double r = __stdlib_erfc(extract(a, i));
            ret = insert(ret, i, r);
        }
        return ret;
    }
}

__declspec(safe) static inline uniform double erfc(uniform double a) { return __stdlib_erfc(a); }

///////////////////////////////////////////////////////////////////////////
// half-precision floats

__declspec(safe) static inline uniform float half_to_float(uniform unsigned int16 h) {
    if (__have_native_half_converts) {
        return __half_to_float_uniform(h);
    } else {
        // https://gist.github.com/2144712
        // Fabian "ryg" Giesen.
        static const uniform unsigned int32 shifted_exp = 0x7c00ul << 13; // exponent mask after shift

        uniform int32 o = ((int32)(h & 0x7fff)) << 13; // exponent/mantissa bits
        uniform unsigned int32 exp = shifted_exp & o;  // just the exponent
        o += (uniform int32)(127 - 15) << 23;          // exponent adjust

        // handle exponent special cases
        if (exp == shifted_exp)                                 // Inf/NaN?
            o += (uniform unsigned int32)(128 - 16) << 23;      // extra exp adjust
        else if (exp == 0) {                                    // Zero/Denormal?
            o += 1ul << 23;                                     // extra exp adjust
            o = intbits(floatbits(o) - floatbits(113ul << 23)); // renormalize
        }

        o |= ((int32)(h & 0x8000)) << 16; // sign bit
        return floatbits(o);
    }
}

__declspec(safe) static inline float half_to_float(unsigned int16 h) {
    if (__have_native_half_converts) {
        return __half_to_float_varying((unsigned int16)h);
    } else {
        // https://gist.github.com/2144712
        // Fabian "ryg" Giesen.

        const unsigned int32 shifted_exp = 0x7c00ul << 13; // exponent mask after shift

        int32 o = ((int32)(h & 0x7ffful)) << 13; // exponent/mantissa bits
        unsigned int32 exp = shifted_exp & o;    // just the exponent
        o += (int32)(127 - 15) << 23;            // exponent adjust

        int32 infnan_val = o + ((int32)(128 - 16) << 23);
        int32 zerodenorm_val = intbits(floatbits(o + (1ul << 23)) - floatbits(113ul << 23));
        int32 reg_val = (exp == 0) ? zerodenorm_val : o;

        int32 sign_bit = ((int32)(h & 0x8000ul)) << 16;
        return floatbits(((exp == shifted_exp) ? infnan_val : reg_val) | sign_bit);
    }
}

__declspec(safe) static inline uniform int16 float_to_half(uniform float f) {
    if (__have_native_half_converts) {
        return __float_to_half_uniform(f);
    } else {
        // via Fabian "ryg" Giesen.
        // https://gist.github.com/2156668
        uniform unsigned int32 sign_mask = 0x80000000u;
        uniform int32 o;

        uniform int32 fint = intbits(f);
        uniform int32 sign = fint & sign_mask;
        fint ^= sign;

        // NOTE all the integer compares in this function can be safely
        // compiled into signed compares since all operands are below
        // 0x80000000. Important if you want fast straight SSE2 code (since
        // there's no unsigned PCMPGTD).

        // Inf or NaN (all exponent bits set)
        // NaN->qNaN and Inf->Inf
        // unconditional assignment here, will override with right value for
        // the regular case below.
        uniform int32 f32infty = 255ul << 23;
        o = (fint > f32infty) ? 0x7e00u : 0x7c00u;

        // (De)normalized number or zero
        // update fint unconditionally to save the blending; we don't need it
        // anymore for the Inf/NaN case anyway.

        const uniform unsigned int32 round_mask = ~0xffful;
        const uniform int32 magic = 15ul << 23;
        const uniform int32 f16infty = 31ul << 23;

        uniform int32 fint2 = intbits(floatbits(fint & round_mask) * floatbits(magic)) - round_mask;
        fint2 = (fint2 > f16infty) ? f16infty : fint2; // Clamp to signed infinity if overflowed

        if (fint < f32infty)
            o = fint2 >> 13; // Take the bits!

        return (o | (sign >> 16));
    }
}

__declspec(safe) static inline int16 float_to_half(float f) {
    if (__have_native_half_converts) {
        return __float_to_half_varying(f);
    } else {
        // via Fabian "ryg" Giesen.
        // https://gist.github.com/2156668
        unsigned int32 sign_mask = 0x80000000u;
        int32 o;

        int32 fint = intbits(f);
        int32 sign = fint & sign_mask;
        fint ^= sign;

        // NOTE all the integer compares in this function can be safely
        // compiled into signed compares since all operands are below
        // 0x80000000. Important if you want fast straight SSE2 code (since
        // there's no unsigned PCMPGTD).

        // Inf or NaN (all exponent bits set)
        // NaN->qNaN and Inf->Inf
        // unconditional assignment here, will override with right value for
        // the regular case below.
        int32 f32infty = 255ul << 23;
        o = (fint > f32infty) ? 0x7e00u : 0x7c00u;

        // (De)normalized number or zero
        // update fint unconditionally to save the blending; we don't need it
        // anymore for the Inf/NaN case anyway.

        const unsigned int32 round_mask = ~0xffful;
        const int32 magic = 15ul << 23;
        const int32 f16infty = 31ul << 23;

        // Shift exponent down, denormalize if necessary.
        // NOTE This represents half-float denormals using single precision denormals.
        // The main reason to do this is that there's no shift with per-lane variable
        // shifts in SSE*, which we'd otherwise need. It has some funky side effects
        // though:
        // - This conversion will actually respect the FTZ (Flush To Zero) flag in
        //   MXCSR - if it's set, no half-float denormals will be generated. I'm
        //   honestly not sure whether this is good or bad. It's definitely interesting.
        // - If the underlying HW doesn't support denormals (not an issue with Intel
        //   CPUs, but might be a problem on GPUs or PS3 SPUs), you will always get
        //   flush-to-zero behavior. This is bad, unless you're on a CPU where you don't
        //   care.
        // - Denormals tend to be slow. FP32 denormals are rare in practice outside of things
        //   like recursive filters in DSP - not a typical half-float application. Whether
        //   FP16 denormals are rare in practice, I don't know. Whatever slow path your HW
        //   may or may not have for denormals, this may well hit it.
        float fscale = floatbits(fint & round_mask) * floatbits(magic);
        fscale = min(fscale, floatbits((31ul << 23) - 0x1000ul));
        int32 fint2 = intbits(fscale) - round_mask;

        if (fint < f32infty)
            o = fint2 >> 13; // Take the bits!

        return (o | (sign >> 16));
    }
}

__declspec(safe) static inline uniform float half_to_float_fast(uniform unsigned int16 h) {
    if (__have_native_half_converts) {
        return __half_to_float_uniform(h);
    } else {
        uniform unsigned int32 hs = h & (int32)0x8000u;  // Pick off sign bit
        uniform unsigned int32 hem = h & (int32)0x7fffu; // Pick off exponent-mantissa bits

        uniform unsigned int32 xs = ((unsigned int32)hs) << 16;
        uniform unsigned int32 xem = ((unsigned int32)hem) << 13;

        xem += 0x38000000; // (127 - 15) << 23

        return floatbits(xs | xem);
    }
}

__declspec(safe) static inline float half_to_float_fast(unsigned int16 h) {
    if (__have_native_half_converts) {
        return __half_to_float_varying(h);
    } else {
        unsigned int32 hs = h & (int32)0x8000u;  // Pick off sign bit
        unsigned int32 hem = h & (int32)0x7fffu; // Pick off exponent-mantissa bits

        unsigned int32 xs = ((unsigned int32)hs) << 16;
        unsigned int32 xem = ((unsigned int32)hem) << 13;

        return floatbits(xs | (xem + 0x38000000 /* (127 - 15) << 23 */));
    }
}

__declspec(safe) static inline uniform int16 float_to_half_fast(uniform float f) {
    if (__have_native_half_converts) {
        return __float_to_half_uniform(f);
    } else {
        uniform int32 x = intbits(f);
        uniform unsigned int32 xs = x & 0x80000000u; // Pick off sign bit
        uniform unsigned int32 xe = x & 0x7F800000u; // Pick off exponent bits
        uniform unsigned int32 xm = x & 0x007FFFFFu; // Pick off mantissa bits

        uniform unsigned int32 hs = (xs >> 16); // Sign bit
        // Exponent unbias the single, then bias the halfp
        uniform int32 hes = ((int)(xe >> 23)) - 127 + 15;
        uniform unsigned int32 he = (hes << 10); // Exponent
        uniform int32 hm = (xm >> 13);           // Mantissa
        uniform int32 ret = (hs | he | hm);

        if (xm & 0x00001000u) // Check for rounding
            // Round, might overflow to inf, this is OK
            ret += 1u;

        return (int16)ret;
    }
}

__declspec(safe) static inline int16 float_to_half_fast(float f) {
    if (__have_native_half_converts) {
        return __float_to_half_varying(f);
    } else {
        int32 x = intbits(f);
        unsigned int32 xs = x & 0x80000000u; // Pick off sign bit
        unsigned int32 xe = x & 0x7F800000u; // Pick off exponent bits
        unsigned int32 xm = x & 0x007FFFFFu; // Pick off mantissa bits

        unsigned int32 hs = (xs >> 16); // Sign bit
        // Exponent unbias the single, then bias the halfp
        int32 hes = ((int)(xe >> 23)) - 127 + 15;
        unsigned int32 he = (hes << 10); // Exponent
        int32 hm = (xm >> 13);           // Mantissa
        int32 ret = (hs | he | hm);

        if (xm & 0x00001000u) // Check for rounding
            // Round, might overflow to inf, this is OK
            ret += 1u;

        return (int16)ret;
    }
}

__attribute__((unmangled)) unmasked uniform float16 __truncsfhf2(uniform float x) {
    return float16bits(float_to_half(x));
}
__attribute__((unmangled)) unmasked uniform float __extendhfsf2(uniform float16 x) { return half_to_float(intbits(x)); }
__attribute__((unmangled)) unmasked uniform float16 __truncdfhf2(uniform double x) {
    return float16bits(float_to_half((uniform float)x));
}
__attribute__((unmangled)) unmasked uniform double __extendhfdf2(uniform float16 x) {
    return (uniform double)half_to_float(intbits(x));
}

///////////////////////////////////////////////////////////////////////////
// float -> srgb8

// https://gist.github.com/2246678, from Fabian "rygorous" Giesen.
//
// The basic ideas are still the same, only this time, we squeeze
// everything into the table, even the linear part of the range; since we
// are approximating the function as piecewise linear anyway, this is
// fairly easy.
//
// In the exact version of the conversion, any value that produces an
// output float less than 0.5 will be rounded to an integer of
// zero. Inverting the linear part of the transform, we get:
//
//   log2(0.5 / (255 * 12.92)) =~ -12.686
//
// which in turn means that any value smaller than about 2^(-12.687) will
// return 0.  What this means is that we can adapt the clamping code to
// just clamp to [2^(-13), 1-eps] and we're covered. This means our table
// needs to cover a range of 13 different exponents from -13 to -1.
//
// The table lookup, storage and interpolation works exactly the same way
// as in the code above.
//
// Max error for the whole function (integer-rounded result minus "exact"
// value, as computed in floats using the official formula): 0.544403 at
// 0x3e9f8000

__declspec(safe) static inline int float_to_srgb8(float inval) {
    static const uniform unsigned int table[104] = {
        0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, 0x00a7001a,
        0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, 0x010e0033, 0x01280033,
        0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033, 0x01dc0067, 0x020f0067, 0x02430067,
        0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067, 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce,
        0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5, 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112,
        0x09940106, 0x0a1700fc, 0x0a9500f2, 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e,
        0x0fbc0150, 0x10630143, 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0,
        0x182401af, 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240,
        0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300, 0x31d105b0,
        0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401, 0x44c20798, 0x488e071e,
        0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559, 0x5e0c0a23, 0x631c0980, 0x67db08f6,
        0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723,
    };

    static const uniform unsigned int near_zero = 0x39000000;
    static const uniform unsigned int almost_one = 0x3f7fffff;

    // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively.
    inval = max(inval, floatbits(near_zero));
    inval = min(inval, floatbits(almost_one));

    // Do the table lookup and unpack bias, scale
    unsigned int tab = table[(intbits(inval) - 0x39000000u) >> 20];
    unsigned int bias = (tab >> 16) << 9;
    unsigned int scale = tab & 0xfffful;

    // Grab next-highest mantissa bits and perform linear interpolation
    unsigned int t = (intbits(inval) >> 12) & 0xff;
    return (bias + scale * t) >> 16;
}

__declspec(safe) static inline uniform int float_to_srgb8(uniform float inval) {
    static const uniform unsigned int table[104] = {
        0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, 0x00a7001a,
        0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, 0x010e0033, 0x01280033,
        0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033, 0x01dc0067, 0x020f0067, 0x02430067,
        0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067, 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce,
        0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5, 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112,
        0x09940106, 0x0a1700fc, 0x0a9500f2, 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e,
        0x0fbc0150, 0x10630143, 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0,
        0x182401af, 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240,
        0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300, 0x31d105b0,
        0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401, 0x44c20798, 0x488e071e,
        0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559, 0x5e0c0a23, 0x631c0980, 0x67db08f6,
        0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723,
    };

    static const uniform unsigned int near_zero = 0x39000000;
    static const uniform unsigned int almost_one = 0x3f7fffff;

    // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively.
    inval = max(inval, floatbits(near_zero));
    inval = min(inval, floatbits(almost_one));

    // Do the table lookup and unpack bias, scale
    uniform unsigned int tab = table[(intbits(inval) - 0x39000000u) >> 20];
    uniform unsigned int bias = (tab >> 16) << 9;
    uniform unsigned int scale = tab & 0xfffful;

    // Grab next-highest mantissa bits and perform linear interpolation
    uniform unsigned int t = (intbits(inval) >> 12) & 0xff;
    return (bias + scale * t) >> 16;
}

///////////////////////////////////////////////////////////////////////////
// srgb8 -> float
//
// It uses a polynomial approximation of (x/255+0.055)/1.055)^2.4 for the range
// [9.384;265.71]. The relative error is 5e-4.
// This should be enough since the input is a 8-bit integer anyway.
// Coefficients are from sollya (with some fine-tunning so 0 <= r <= 1.0).
//
// For more precise results a (1KiB) LUT can be used instead.
// Using a LUT is generally (1~2 times) slower on AVX2/AVX-512 targets.
// However, using a LUT might be faster on SSE/Neon with a warm cache.

__declspec(safe) static inline float srgb8_to_float(int inval) {
    const float x = clamp(inval, (uniform int)0, (uniform int)255);
    float r = 1.0489326014e-13;
    r = r * x - 8.6331650162e-11;
    r = r * x + 4.2816203915e-8;
    r = r * x + 7.7981030699e-6;
    r = r * x + 1.3343113823e-4;
    r = r * x + 8.7884825188e-4;
    const float scale = 1.0 / 255 / 12.92;
    return select(x < 10.31475, x * scale, r);
}

__declspec(safe) static inline uniform float srgb8_to_float(uniform int inval) {
    const uniform float x = clamp(inval, (uniform int)0, (uniform int)255);
    uniform float r = 1.0489326014e-13;
    r = r * x - 8.6331650162e-11;
    r = r * x + 4.2816203915e-8;
    r = r * x + 7.7981030699e-6;
    r = r * x + 1.3343113823e-4;
    r = r * x + 8.7884825188e-4;
    const uniform float scale = 1.0 / 255 / 12.92;
    return select(x < 10.31475, x * scale, r);
}

///////////////////////////////////////////////////////////////////////////
// RNG stuff

static inline unsigned int random(varying RNGState *uniform state) {
    unsigned int b;

    b = ((state->z1 << 6) ^ state->z1) >> 13;
    state->z1 = ((state->z1 & 4294967294U) << 18) ^ b;
    b = ((state->z2 << 2) ^ state->z2) >> 27;
    state->z2 = ((state->z2 & 4294967288U) << 2) ^ b;
    b = ((state->z3 << 13) ^ state->z3) >> 21;
    state->z3 = ((state->z3 & 4294967280U) << 7) ^ b;
    b = ((state->z4 << 3) ^ state->z4) >> 12;
    state->z4 = ((state->z4 & 4294967168U) << 13) ^ b;
    return (state->z1 ^ state->z2 ^ state->z3 ^ state->z4);
}

static inline uniform unsigned int random(uniform RNGState *uniform state) {
    uniform unsigned int b;

    b = ((state->z1 << 6) ^ state->z1) >> 13;
    state->z1 = ((state->z1 & 4294967294U) << 18) ^ b;
    b = ((state->z2 << 2) ^ state->z2) >> 27;
    state->z2 = ((state->z2 & 4294967288U) << 2) ^ b;
    b = ((state->z3 << 13) ^ state->z3) >> 21;
    state->z3 = ((state->z3 & 4294967280U) << 7) ^ b;
    b = ((state->z4 << 3) ^ state->z4) >> 12;
    state->z4 = ((state->z4 & 4294967168U) << 13) ^ b;
    return (state->z1 ^ state->z2 ^ state->z3 ^ state->z4);
}

static inline float frandom(varying RNGState *uniform state) {
    unsigned int irand = random(state);
    irand &= (1ul << 23) - 1;
    return floatbits(0x3F800000 | irand) - 1.0f;
}

static inline uniform float frandom(uniform RNGState *uniform state) {
    uniform unsigned int irand = random(state);
    irand &= (1ul << 23) - 1;
    return floatbits(0x3F800000 | irand) - 1.0f;
}

static inline void seed_rng(varying RNGState *uniform state, unsigned int seed) {
    state->z1 = seed;
    state->z2 = seed ^ 0xbeeff00d;
    state->z3 = ((seed & 0xfffful) << 16) | (seed >> 16);
    state->z4 =
        (((seed & 0xfful) << 24) | ((seed & 0xff00ul) << 8) | ((seed & 0xff0000ul) >> 8) | (seed & 0xff000000ul) >> 24);
}

static inline void seed_rng(uniform RNGState *uniform state, uniform unsigned int seed) {
    state->z1 = seed;
    state->z2 = seed ^ 0xbeeff00d;
    state->z3 = ((seed & 0xfffful) << 16) | (seed >> 16);
    state->z4 =
        (((seed & 0xfful) << 24) | ((seed & 0xff00ul) << 8) | ((seed & 0xff0000ul) >> 8) | (seed & 0xff000000ul) >> 24);
}

static inline void fastmath() { __fastmath(); }

///////////////////////////////////////////////////////////////////////////
// saturation arithmetic

static inline uniform int8 saturating_add(uniform int8 a, uniform int8 b) {
    if (__have_saturating_arithmetic) {
        return __padds_ui8(a, b);
    } else {
        uniform unsigned int8 a_unsig = a, b_unsig = b;
        uniform unsigned int8 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 7) + INT8_MAX;
        if ((uniform int8)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int8 saturating_add(varying int8 a, varying int8 b) { return __padds_vi8(a, b); }

static inline uniform int16 saturating_add(uniform int16 a, uniform int16 b) {
    if (__have_saturating_arithmetic) {
        return __padds_ui16(a, b);
    } else {
        uniform unsigned int16 a_unsig = a, b_unsig = b;
        uniform unsigned int16 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 15) + INT16_MAX;
        if ((uniform int16)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int16 saturating_add(varying int16 a, varying int16 b) { return __padds_vi16(a, b); }

static inline uniform int32 saturating_add(uniform int32 a, uniform int32 b) {
    if (__have_saturating_arithmetic) {
        return __padds_ui32(a, b);
    } else {
        uniform unsigned int32 a_unsig = a, b_unsig = b;
        uniform unsigned int32 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 31) + INT32_MAX;
        if ((uniform int32)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int32 saturating_add(varying int32 a, varying int32 b) {
    if (__have_saturating_arithmetic) {
        return __padds_vi32(a, b);
    } else {
        varying unsigned int32 a_unsig = a, b_unsig = b;
        varying unsigned int32 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 31) + INT32_MAX;
        if ((varying int32)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline uniform int64 saturating_add(uniform int64 a, uniform int64 b) {
    if (__have_saturating_arithmetic) {
        return __padds_ui64(a, b);
    } else {
        uniform unsigned int64 a_unsig = a, b_unsig = b;
        uniform unsigned int64 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 63) + INT64_MAX;
        if ((uniform int64)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int64 saturating_add(varying int64 a, varying int64 b) {
    if (__have_saturating_arithmetic) {
        return __padds_vi64(a, b);
    } else {
        varying unsigned int64 a_unsig = a, b_unsig = b;
        varying unsigned int64 result = a_unsig + b_unsig;
        a_unsig = (a_unsig >> 63) + INT64_MAX;
        if ((varying int64)((a_unsig ^ b_unsig) | ~(b_unsig ^ result)) >= 0)
            result = a_unsig;
        return result;
    }
}

static inline uniform unsigned int8 saturating_add(uniform unsigned int8 a, uniform unsigned int8 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_ui8(a, b);
    } else {
        uniform unsigned int8 result = a + b;
        result |= (-(uniform int8)(result < a));
        return result;
    }
}

static inline varying unsigned int8 saturating_add(varying unsigned int8 a, varying unsigned int8 b) {
    return __paddus_vi8(a, b);
}

static inline uniform unsigned int16 saturating_add(uniform unsigned int16 a, uniform unsigned int16 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_ui16(a, b);
    } else {
        uniform unsigned int16 result = a + b;
        result |= (-(uniform int16)(result < a));
        return result;
    }
}

static inline varying unsigned int16 saturating_add(varying unsigned int16 a, varying unsigned int16 b) {
    return __paddus_vi16(a, b);
}

static inline uniform unsigned int32 saturating_add(uniform unsigned int32 a, uniform unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_ui32(a, b);
    } else {
        uniform unsigned int32 result = a + b;
        result |= (-(uniform int32)(result < a));
        return result;
    }
}

static inline varying unsigned int32 saturating_add(varying unsigned int32 a, varying unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_vi32(a, b);
    } else {
        varying unsigned int32 result = a + b;
        result |= (-(varying int32)(result < a));
        return result;
    }
}

static inline uniform unsigned int64 saturating_add(uniform unsigned int64 a, uniform unsigned int64 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_ui64(a, b);
    } else {
        uniform unsigned int64 result = a + b;
        result |= (-(uniform int64)(result < a));
        return result;
    }
}

static inline varying unsigned int64 saturating_add(varying unsigned int64 a, varying unsigned int64 b) {
    if (__have_saturating_arithmetic) {
        return __paddus_vi64(a, b);
    } else {
        varying unsigned int64 result = a + b;
        result |= (-(varying int64)(result < a));
        return result;
    }
}

static inline uniform int8 saturating_sub(uniform int8 a, uniform int8 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_ui8(a, b);
    } else {
        uniform unsigned int8 a_unsig = a, b_unsig = b;
        uniform unsigned int8 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 7) + INT8_MAX;
        if ((uniform int8)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int8 saturating_sub(varying int8 a, varying int8 b) { return __psubs_vi8(a, b); }

static inline uniform int16 saturating_sub(uniform int16 a, uniform int16 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_ui16(a, b);
    } else {
        uniform unsigned int16 a_unsig = a, b_unsig = b;
        uniform unsigned int16 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 15) + INT16_MAX;
        if ((uniform int16)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int16 saturating_sub(varying int16 a, varying int16 b) { return __psubs_vi16(a, b); }

static inline uniform int32 saturating_sub(uniform int32 a, uniform int32 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_ui32(a, b);
    } else {
        uniform unsigned int32 a_unsig = a, b_unsig = b;
        uniform unsigned int32 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 31) + INT32_MAX;
        if ((uniform int32)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int32 saturating_sub(varying int32 a, varying int32 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_vi32(a, b);
    } else {
        varying unsigned int32 a_unsig = a, b_unsig = b;
        varying unsigned int32 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 31) + INT32_MAX;
        if ((varying int32)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline uniform int64 saturating_sub(uniform int64 a, uniform int64 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_ui64(a, b);
    } else {
        uniform unsigned int64 a_unsig = a, b_unsig = b;
        uniform unsigned int64 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 63) + INT64_MAX;
        if ((uniform int64)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline varying int64 saturating_sub(varying int64 a, varying int64 b) {
    if (__have_saturating_arithmetic) {
        return __psubs_vi64(a, b);
    } else {
        varying unsigned int64 a_unsig = a, b_unsig = b;
        varying unsigned int64 result = a_unsig - b_unsig;
        a_unsig = (a_unsig >> 63) + INT64_MAX;
        if ((varying int64)((a_unsig ^ b_unsig) & (a_unsig ^ result)) < 0)
            result = a_unsig;
        return result;
    }
}

static inline uniform unsigned int8 saturating_sub(uniform unsigned int8 a, uniform unsigned int8 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_ui8(a, b);
    } else {
        uniform unsigned int8 result = a - b;
        result &= (-(uniform int8)(result <= a));
        return result;
    }
}

static inline varying unsigned int8 saturating_sub(varying unsigned int8 a, varying unsigned int8 b) {
    return __psubus_vi8(a, b);
}

static inline uniform unsigned int16 saturating_sub(uniform unsigned int16 a, uniform unsigned int16 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_ui16(a, b);
    } else {
        uniform unsigned int16 result = a - b;
        result &= (-(uniform int16)(result <= a));
        return result;
    }
}

static inline varying unsigned int16 saturating_sub(varying unsigned int16 a, varying unsigned int16 b) {
    return __psubus_vi16(a, b);
}

static inline uniform unsigned int32 saturating_sub(uniform unsigned int32 a, uniform unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_ui32(a, b);
    } else {
        uniform unsigned int32 result = a - b;
        result &= (-(uniform int32)(result <= a));
        return result;
    }
}

static inline varying unsigned int32 saturating_sub(varying unsigned int32 a, varying unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_vi32(a, b);
    } else {
        varying unsigned int32 result = a - b;
        result &= (-(varying int32)(result <= a));
        return result;
    }
}

static inline uniform unsigned int64 saturating_sub(uniform unsigned int64 a, uniform unsigned int64 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_ui64(a, b);
    } else {
        uniform unsigned int64 result = a - b;
        result &= (-(uniform int64)(result <= a));
        return result;
    }
}

static inline varying unsigned int64 saturating_sub(varying unsigned int64 a, varying unsigned int64 b) {
    if (__have_saturating_arithmetic) {
        return __psubus_vi64(a, b);
    } else {
        varying unsigned int64 result = a - b;
        result &= (-(varying int64)(result <= a));
        return result;
    }
}

static inline uniform int8 saturating_div(uniform int8 a, uniform int8 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((uniform unsigned int8)a + INT8_MIN));
    return a / b;
}

static inline varying int8 saturating_div(varying int8 a, varying int8 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((varying unsigned int8)a + INT8_MIN));
    return a / b;
}

static inline uniform int16 saturating_div(uniform int16 a, uniform int16 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((uniform unsigned int16)a + INT16_MIN));
    return a / b;
}

static inline varying int16 saturating_div(varying int16 a, varying int16 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((varying unsigned int16)a + INT16_MIN));
    return a / b;
}

static inline uniform int32 saturating_div(uniform int32 a, uniform int32 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((uniform unsigned int32)a + INT32_MIN));
    return a / b;
}

static inline varying int32 saturating_div(varying int32 a, varying int32 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((varying unsigned int32)a + INT32_MIN));
    return a / b;
}

static inline uniform int64 saturating_div(uniform int64 a, uniform int64 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((uniform unsigned int64)a + INT64_MIN));
    return a / b;
}

static inline varying int64 saturating_div(varying int64 a, varying int64 b) {
    /* Only one way to overflow, so test for and prevent it. */
    a += !((b + 1) | ((varying unsigned int64)a + INT64_MIN));
    return a / b;
}

static inline uniform unsigned int8 saturating_div(uniform unsigned int8 a, uniform unsigned int8 b) {
    /* No overflow possible */
    return a / b;
}

static inline varying unsigned int8 saturating_div(varying unsigned int8 a, varying unsigned int8 b) {
    /* No overflow possible */
    return a / b;
}

static inline uniform unsigned int16 saturating_div(uniform unsigned int16 a, uniform unsigned int16 b) {
    /* No overflow possible */
    return a / b;
}

static inline varying unsigned int16 saturating_div(varying unsigned int16 a, varying unsigned int16 b) {
    /* No overflow possible */
    return a / b;
}

static inline uniform unsigned int32 saturating_div(uniform unsigned int32 a, uniform unsigned int32 b) {
    /* No overflow possible */
    return a / b;
}

static inline varying unsigned int32 saturating_div(varying unsigned int32 a, varying unsigned int32 b) {
    /* No overflow possible */
    return a / b;
}

static inline uniform unsigned int64 saturating_div(uniform unsigned int64 a, uniform unsigned int64 b) {
    /* No overflow possible */
    return a / b;
}

static inline varying unsigned int64 saturating_div(varying unsigned int64 a, varying unsigned int64 b) {
    /* No overflow possible */
    return a / b;
}

static inline uniform int8 saturating_mul(uniform int8 a, uniform int8 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_ui8(a, b);
    } else {
        uniform int16 result = (uniform int16)a * (uniform int16)b;
        uniform unsigned int8 result2 = ((uniform unsigned int8)(a ^ b) >> 7) + INT8_MAX;
        uniform int8 hi = result >> 8;
        uniform int8 lo = result;
        if (hi != (lo >> 7))
            result = result2;
        return result;
    }
}

static inline varying int8 saturating_mul(varying int8 a, varying int8 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_vi8(a, b);
    } else {
        varying int16 result = (varying int16)a * (varying int16)b;
        varying unsigned int8 result2 = ((varying unsigned int8)(a ^ b) >> 7) + INT8_MAX;
        varying int8 hi = result >> 8;
        varying int8 lo = result;
        if (hi != (lo >> 7))
            result = result2;
        return result;
    }
}

static inline uniform int16 saturating_mul(uniform int16 a, uniform int16 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_ui16(a, b);
    } else {
        uniform int32 result = (uniform int32)a * (uniform int32)b;
        uniform unsigned int16 result2 = ((uniform unsigned int16)(a ^ b) >> 15) + INT16_MAX;
        uniform int16 hi = result >> 16;
        uniform int16 lo = result;
        if (hi != (lo >> 15))
            result = result2;
        return result;
    }
}

static inline varying int16 saturating_mul(varying int16 a, varying int16 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_vi16(a, b);
    } else {
        varying int32 result = (varying int32)a * (varying int32)b;
        varying unsigned int16 result2 = ((varying unsigned int16)(a ^ b) >> 15) + INT16_MAX;
        varying int16 hi = result >> 16;
        varying int16 lo = result;
        if (hi != (lo >> 15))
            result = result2;
        return result;
    }
}

static inline uniform int32 saturating_mul(uniform int32 a, uniform int32 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_ui32(a, b);
    } else {
        uniform int64 result = (uniform int64)a * (uniform int64)b;
        uniform unsigned int32 result2 = ((uniform unsigned int32)(a ^ b) >> 31) + INT32_MAX;
        uniform int32 hi = result >> 32;
        uniform int32 lo = result;
        if (hi != (lo >> 31))
            result = result2;
        return result;
    }
}

static inline varying int32 saturating_mul(varying int32 a, varying int32 b) {
    if (__have_saturating_arithmetic) {
        return __pmuls_vi32(a, b);
    } else {
        varying int64 result = (varying int64)a * (varying int64)b;
        varying unsigned int32 result2 = ((varying unsigned int32)(a ^ b) >> 31) + INT32_MAX;
        varying int32 hi = result >> 32;
        varying int32 lo = result;
        if (hi != (lo >> 31))
            result = result2;
        return result;
    }
}

static inline uniform unsigned int8 saturating_mul(uniform unsigned int8 a, uniform unsigned int8 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_ui8(a, b);
    } else {
        uniform unsigned int16 result = (uniform unsigned int16)a * (uniform unsigned int16)b;
        uniform unsigned int8 hi = result >> 8;
        uniform unsigned int8 lo = result;
        return lo | -(uniform int8) !!hi;
    }
}

static inline varying unsigned int8 saturating_mul(varying unsigned int8 a, varying unsigned int8 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_vi8(a, b);
    } else {
        varying unsigned int16 result = (varying unsigned int16)a * (varying unsigned int16)b;
        varying unsigned int8 hi = result >> 8;
        varying unsigned int8 lo = result;
        return lo | -(varying int8) !!hi;
    }
}

static inline uniform unsigned int16 saturating_mul(uniform unsigned int16 a, uniform unsigned int16 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_ui16(a, b);
    } else {
        uniform unsigned int32 result = (uniform unsigned int32)a * (uniform unsigned int32)b;
        uniform unsigned int16 hi = result >> 16;
        uniform unsigned int16 lo = result;
        return lo | -(uniform int16) !!hi;
    }
}

static inline varying unsigned int16 saturating_mul(varying unsigned int16 a, varying unsigned int16 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_vi16(a, b);
    } else {
        varying unsigned int32 result = (varying unsigned int32)a * (varying unsigned int32)b;
        varying unsigned int16 hi = result >> 16;
        varying unsigned int16 lo = result;
        return lo | -(varying int16) !!hi;
    }
}

static inline uniform unsigned int32 saturating_mul(uniform unsigned int32 a, uniform unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_ui32(a, b);
    } else {
        uniform unsigned int64 result = (uniform unsigned int64)a * (uniform unsigned int64)b;
        uniform unsigned int32 hi = result >> 32;
        uniform unsigned int32 lo = result;
        return lo | -(uniform int32) !!hi;
    }
}

static inline varying unsigned int32 saturating_mul(varying unsigned int32 a, varying unsigned int32 b) {
    if (__have_saturating_arithmetic) {
        return __pmulus_vi32(a, b);
    } else {
        varying unsigned int64 result = (varying unsigned int64)a * (varying unsigned int64)b;
        varying unsigned int32 hi = result >> 32;
        varying unsigned int32 lo = result;
        return lo | -(varying int32) !!hi;
    }
}

static inline uniform int64 saturating_mul(uniform int64 a, uniform int64 b) {
    uniform unsigned int64 ret = 0;

    uniform int8 sign = (((a > 0) && (b > 0)) || ((a < 0) && (b < 0))) ? 1 : -1;
    uniform unsigned int64 a_abs = 0;
    uniform unsigned int64 b_abs = 0;

    if (a == INT64_MIN)
        // Operation "-" is undefined for "INT64_MIN", as it causes overflow.
        // But converting INT64_MIN to unsigned type yields the correct result,
        // i.e. it will be positive value -INT64_MIN.
        // See 6.3.1.3 section in C99 standart for more details (ISPC follows
        // C standard, unless it's specifically different in the language).
        a_abs = (uniform unsigned int64)INT64_MIN;
    else
        a_abs = (a > 0) ? a : -a;

    if (b == INT64_MIN)
        b_abs = (uniform unsigned int64)INT64_MIN;
    else
        b_abs = (b > 0) ? b : -b;

    uniform unsigned int32 a0 = a_abs & 0xFFFFFFFF;
    uniform unsigned int32 b0 = b_abs & 0xFFFFFFFF;
    uniform unsigned int32 a1 = a_abs >> 32;
    uniform unsigned int32 b1 = b_abs >> 32;

    if ((a1 != 0) && (b1 != 0)) {
        if (sign > 0) {
            return INT64_MAX;
        } else {
            return INT64_MIN;
        }
    } else if (a1 != 0) {
        ret = saturating_add((uniform unsigned int64)saturating_mul(b0, a1) << 32, (uniform unsigned int64)(a0)*b0);
    } else if (b1 != 0) {
        ret = saturating_add((uniform unsigned int64)saturating_mul(a0, b1) << 32, (uniform unsigned int64)(a0)*b0);
    } else {
        ret = a_abs * b_abs;
    }

    if ((sign < 0) && (ret >= (uniform unsigned int64)INT64_MIN)) {
        return INT64_MIN;
    } else if ((sign > 0) && (ret >= INT64_MAX)) {
        return INT64_MAX;
    } else {
        return ret * sign;
    }
}

static inline varying int64 saturating_mul(varying int64 a, varying int64 b) {
    varying unsigned int64 ret = 0;

    varying int8 sign = (((a > 0) && (b > 0)) || ((a < 0) && (b < 0))) ? 1 : -1;
    varying unsigned int64 a_abs = 0;
    varying unsigned int64 b_abs = 0;

    if (a == INT64_MIN)
        // Operation "-" is undefined for "INT64_MIN", as it causes overflow.
        // But converting INT64_MIN to unsigned type yields the correct result,
        // i.e. it will be positive value -INT64_MIN.
        // See 6.3.1.3 section in C99 standart for more details (ISPC follows
        // C standard, unless it's specifically different in the language).
        a_abs = (varying unsigned int64)INT64_MIN;
    else
        a_abs = (a > 0) ? a : -a;

    if (b == INT64_MIN)
        b_abs = (varying unsigned int64)INT64_MIN;
    else
        b_abs = (b > 0) ? b : -b;

    varying unsigned int32 a0 = a_abs & 0xFFFFFFFF;
    varying unsigned int32 b0 = b_abs & 0xFFFFFFFF;
    varying unsigned int32 a1 = a_abs >> 32;
    varying unsigned int32 b1 = b_abs >> 32;

    if ((a1 != 0) && (b1 != 0)) {
        if (sign > 0) {
            return INT64_MAX;
        } else {
            return INT64_MIN;
        }
    } else if (a1 != 0) {
        ret = saturating_add((varying unsigned int64)saturating_mul(b0, a1) << 32, (varying unsigned int64)(a0)*b0);
    } else if (b1 != 0) {
        ret = saturating_add((varying unsigned int64)saturating_mul(a0, b1) << 32, (varying unsigned int64)(a0)*b0);
    } else {
        ret = a_abs * b_abs;
    }

    if ((sign < 0) && (ret >= (varying unsigned int64)INT64_MIN)) {
        return INT64_MIN;
    } else if ((sign > 0) && (ret >= INT64_MAX)) {
        return INT64_MAX;
    } else {
        return ret * sign;
    }
}

static inline uniform unsigned int64 saturating_mul(uniform unsigned int64 a, uniform unsigned int64 b) {

    uniform unsigned int32 a0 = a & 0xFFFFFFFF;
    uniform unsigned int32 b0 = b & 0xFFFFFFFF;
    uniform unsigned int32 a1 = a >> 32;
    uniform unsigned int32 b1 = b >> 32;

    if ((a1 != 0) && (b1 != 0)) {
        return UINT64_MAX;
    } else if (a1 != 0) {
        return saturating_add((uniform unsigned int64)saturating_mul(b0, a1) << 32, (uniform unsigned int64)(a0)*b0);
    } else if (b1 != 0) {
        return saturating_add((uniform unsigned int64)saturating_mul(a0, b1) << 32, (uniform unsigned int64)(a0)*b0);
    } else {
        return a * b;
    }
}

static inline varying unsigned int64 saturating_mul(varying unsigned int64 a, varying unsigned int64 b) {
    varying unsigned int32 a0 = a & 0xFFFFFFFF;
    varying unsigned int32 b0 = b & 0xFFFFFFFF;
    varying unsigned int32 a1 = a >> 32;
    varying unsigned int32 b1 = b >> 32;

    if ((a1 != 0) && (b1 != 0)) {
        return UINT64_MAX;
    } else if (a1 != 0) {
        return saturating_add((varying unsigned int64)saturating_mul(b0, a1) << 32, (varying unsigned int64)(a0)*b0);
    } else if (b1 != 0) {
        return saturating_add((varying unsigned int64)saturating_mul(a0, b1) << 32, (varying unsigned int64)(a0)*b0);
    } else {
        return a * b;
    }
}

///////////////////////////////////////////////////////////////////////////
// rdrand

static inline uniform bool rdrand(float *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        uniform int32 irand;
        uniform bool success = __rdrand_i32((opaque_ptr_t)&irand);
        if (success) {
            irand &= (1ul << 23) - 1;
            *ptr = floatbits(0x3F800000 | irand) - 1.0f;
        }
        return success;
    }
}

static inline bool rdrand(varying float *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        bool success = false;
        foreach_active(index) {
            uniform int32 irand;
            if (__rdrand_i32((opaque_ptr_t)&irand)) {
                // FIXME: it probably would be preferable, here and in the
                // following rdrand() function, to do the int->float stuff
                // in vector form.  However, we need to be careful to not
                // clobber any existing already-set values in *ptr with
                // inactive lanes here...
                irand &= (1ul << 23) - 1;
                *ptr = floatbits(0x3F800000 | irand) - 1.0f;
                success = true;
            }
        }
        return success;
    }
}

static inline bool rdrand(float *ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        float *uniform ptrs[programCount];
        ptrs[programIndex] = ptr;

        bool success = false;
        foreach_active(index) {
            uniform int32 irand;
            if (__rdrand_i32((opaque_ptr_t)&irand)) {
                irand &= (1ul << 23) - 1;
                *ptrs[index] = floatbits(0x3F800000 | irand) - 1.0f;
                success = true;
            }
        }
        return success;
    }
}

static inline uniform bool rdrand(int16 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else
        return __rdrand_i16((uniform int8 * uniform) ptr);
}

static inline bool rdrand(varying int16 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        bool success = false;
        foreach_active(index) {
            uniform int16 irand;
            if (__rdrand_i16((opaque_ptr_t)&irand)) {
                *ptr = irand;
                success = true;
            }
        }
        return success;
    }
}

static inline bool rdrand(int16 *ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        int16 *uniform ptrs[programCount];
        ptrs[programIndex] = ptr;
        bool success = false;

        foreach_active(index) {
            uniform int16 irand;
            if (__rdrand_i16((opaque_ptr_t)&irand)) {
                *ptrs[index] = irand;
                success = true;
            }
        }
        return success;
    }
}

static inline uniform bool rdrand(int32 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else
        return __rdrand_i32((uniform int8 * uniform) ptr);
}

static inline bool rdrand(varying int32 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        bool success = false;
        foreach_active(index) {
            uniform int32 irand;
            if (__rdrand_i32((opaque_ptr_t)&irand)) {
                *ptr = irand;
                success = true;
            }
        }
        return success;
    }
}

static inline bool rdrand(int32 *ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        int32 *uniform ptrs[programCount];
        ptrs[programIndex] = ptr;
        bool success = false;

        foreach_active(index) {
            uniform int32 irand;
            if (__rdrand_i32((opaque_ptr_t)&irand)) {
                *ptrs[index] = irand;
                success = true;
            }
        }
        return success;
    }
}

static inline uniform bool rdrand(int64 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else
        return __rdrand_i64((uniform int8 * uniform) ptr);
}

static inline bool rdrand(varying int64 *uniform ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        bool success = false;
        foreach_active(index) {
            uniform int64 irand;
            if (__rdrand_i64((opaque_ptr_t)&irand)) {
                *ptr = irand;
                success = true;
            }
        }
        return success;
    }
}

static inline bool rdrand(int64 *ptr) {
    if (__have_native_rand == false)
        return false;
    else {
        int64 *uniform ptrs[programCount];
        ptrs[programIndex] = ptr;
        bool success = false;

        foreach_active(index) {
            uniform int64 irand;
            if (__rdrand_i64((opaque_ptr_t)&irand)) {
                *ptrs[index] = irand;
                success = true;
            }
        }
        return success;
    }
}

///////////////////////////////////////////////////////////////////////////
// Saturating int8/int16 ops

__declspec(safe) static unmasked inline unsigned int8 avg_up(unsigned int8 a, unsigned int8 b) {
    return __avg_up_uint8(a, b);
}

__declspec(safe) static unmasked inline int8 avg_up(int8 a, int8 b) { return __avg_up_int8(a, b); }

__declspec(safe) static unmasked inline unsigned int16 avg_up(unsigned int16 a, unsigned int16 b) {
    return __avg_up_uint16(a, b);
}

__declspec(safe) static unmasked inline int16 avg_up(int16 a, int16 b) { return __avg_up_int16(a, b); }

__declspec(safe) static unmasked inline unsigned int8 avg_down(unsigned int8 a, unsigned int8 b) {
    return __avg_down_uint8(a, b);
}

__declspec(safe) static unmasked inline int8 avg_down(int8 a, int8 b) { return __avg_down_int8(a, b); }

__declspec(safe) static unmasked inline unsigned int16 avg_down(unsigned int16 a, unsigned int16 b) {
    return __avg_down_uint16(a, b);
}

__declspec(safe) static unmasked inline int16 avg_down(int16 a, int16 b) { return __avg_down_int16(a, b); }

///////////////////////////////////////////////////////////////////////////
// Assume uniform/varying ops
__declspec(safe) static inline void assume(uniform bool test) {
    __do_assume_uniform(test);
    return;
}

///////////////////////////////////////////////////////////////////////////
// Dot product and accumulate

// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed
// 8-bit integers in b, producing 4 intermediate signed 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc, and return the result.
__declspec(safe) static inline varying int32 dot4add_u8i8packed(varying uint32 a, varying uint32 b, varying int32 acc) {
    if (__have_intel_vnni || __have_arm_i8mm) {
        return __dot4add_u8i8packed(a, b, acc);
    } else {
        int16 tmp1 = (int16)((a >> 24) & 0xFF) * (int16)((int8)((b >> 24) & 0xFF));
        int16 tmp2 = (int16)((a >> 16) & 0xFF) * (int16)((int8)((b >> 16) & 0xFF));
        int16 tmp3 = (int16)((a >> 8) & 0xFF) * (int16)((int8)((b >> 8) & 0xFF));
        int16 tmp4 = (int16)(a & 0xFF) * (int16)((int8)(b & 0xFF));
        return (int32)tmp1 + (int32)tmp2 + (int32)tmp3 + (int32)tmp4 + acc;
    }
}

// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed
// 8-bit integers in b, producing 4 intermediate signed 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc using signed saturation, and return the result.
__declspec(safe) static inline varying int32 dot4add_u8i8packed_sat(varying uint32 a, varying uint32 b,
                                                                    varying int32 acc) {
    if (__have_intel_vnni) {
        return __dot4add_u8i8packed_sat(a, b, acc);
    } else {
        int16 tmp1 = (int16)((a >> 24) & 0xFF) * (int16)((int8)((b >> 24) & 0xFF));
        int16 tmp2 = (int16)((a >> 16) & 0xFF) * (int16)((int8)((b >> 16) & 0xFF));
        int16 tmp3 = (int16)((a >> 8) & 0xFF) * (int16)((int8)((b >> 8) & 0xFF));
        int16 tmp4 = (int16)(a & 0xFF) * (int16)((int8)(b & 0xFF));
        int32 tmp1234 = ((int32)tmp1) + ((int32)tmp2) + ((int32)tmp3) + ((int32)tmp4);
        return saturating_add(tmp1234, acc);
    }
}

// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding unsigned
// 8-bit integers in b, producing 4 intermediate unsigned 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc, and return the result.
__declspec(safe) static inline varying uint32 dot4add_u8u8packed(varying uint32 a, varying uint32 b,
                                                                 varying uint32 acc) {
    if (__have_arm_dot_product || __have_intel_vnni_int8) {
        return __dot4add_u8u8packed(a, b, acc);
    } else {
        int16 tmp1 = (int16)((a >> 24) & 0xFF) * (int16)((b >> 24) & 0xFF);
        int16 tmp2 = (int16)((a >> 16) & 0xFF) * (int16)((b >> 16) & 0xFF);
        int16 tmp3 = (int16)((a >> 8) & 0xFF) * (int16)((b >> 8) & 0xFF);
        int16 tmp4 = (int16)(a & 0xFF) * (int16)(b & 0xFF);
        return (uint32)tmp1 + (uint32)tmp2 + (uint32)tmp3 + (uint32)tmp4 + acc;
    }
}

// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding unsigned
// 8-bit integers in b, producing 4 intermediate unsigned 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc using unsigned saturation, and return the result.
__declspec(safe) static inline varying uint32 dot4add_u8u8packed_sat(varying uint32 a, varying uint32 b,
                                                                     varying uint32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot4add_u8u8packed_sat(a, b, acc);
    } else {
        uint16 tmp1 = (uint16)((a >> 24) & 0xFF) * (uint16)((b >> 24) & 0xFF);
        uint16 tmp2 = (uint16)((a >> 16) & 0xFF) * (uint16)((b >> 16) & 0xFF);
        uint16 tmp3 = (uint16)((a >> 8) & 0xFF) * (uint16)((b >> 8) & 0xFF);
        uint16 tmp4 = (uint16)(a & 0xFF) * (uint16)(b & 0xFF);
        uint32 tmp1234 = ((uint32)tmp1) + ((uint32)tmp2) + ((uint32)tmp3) + ((uint32)tmp4);
        return saturating_add(tmp1234, acc);
    }
}

// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed
// 8-bit integers in b, producing 4 intermediate signed 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc, and return the result.
__declspec(safe) static inline varying int32 dot4add_i8i8packed(varying uint32 a, varying uint32 b, varying int32 acc) {
    if (__have_arm_dot_product || __have_intel_vnni_int8) {
        return __dot4add_i8i8packed(a, b, acc);
    } else {
        int16 tmp1 = (int16)((int8)((a >> 24) & 0xFF)) * (int16)((int8)((b >> 24) & 0xFF));
        int16 tmp2 = (int16)((int8)((a >> 16) & 0xFF)) * (int16)((int8)((b >> 16) & 0xFF));
        int16 tmp3 = (int16)((int8)((a >> 8) & 0xFF)) * (int16)((int8)((b >> 8) & 0xFF));
        int16 tmp4 = (int16)((int8)(a & 0xFF)) * (int16)((int8)(b & 0xFF));
        return (int32)tmp1 + (int32)tmp2 + (int32)tmp3 + (int32)tmp4 + acc;
    }
}

// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding signed
// 8-bit integers in b, producing 4 intermediate signed 16-bit results.
// Sum these 4 results with the corresponding 32-bit integer in acc using signed saturation, and return the result.
__declspec(safe) static inline varying int32 dot4add_i8i8packed_sat(varying uint32 a, varying uint32 b,
                                                                    varying int32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot4add_i8i8packed_sat(a, b, acc);
    } else {
        int16 tmp1 = (int16)((a >> 24) & 0xFF) * (int16)((b >> 24) & 0xFF);
        int16 tmp2 = (int16)((a >> 16) & 0xFF) * (int16)((b >> 16) & 0xFF);
        int16 tmp3 = (int16)((a >> 8) & 0xFF) * (int16)((b >> 8) & 0xFF);
        int16 tmp4 = (int16)(a & 0xFF) * (int16)(b & 0xFF);
        int32 tmp1234 = ((int32)tmp1) + ((int32)tmp2) + ((int32)tmp3) + ((int32)tmp4);
        return saturating_add(tmp1234, acc);
    }
}

// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a
// with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in src, and return the result.
__declspec(safe) static inline varying int32 dot2add_i16i16packed(varying uint32 a, varying uint32 b,
                                                                  varying int32 acc) {
    if (__have_intel_vnni) {
        return __dot2add_i16i16packed(a, b, acc);
    } else {
        int32 tmp1 = (int32)((int16)((a >> 16) & 0xFFFF)) * (int32)((int16)((b >> 16) & 0xFFFF));
        int32 tmp2 = (int32)((int16)(a & 0xFFFF)) * (int32)((int16)(b & 0xFFFF));
        return tmp1 + tmp2 + acc;
    }
}

// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a
// with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and return the result.
__declspec(safe) static inline varying int32 dot2add_i16i16packed_sat(varying uint32 a, varying uint32 b,
                                                                      varying int32 acc) {
    if (__have_intel_vnni) {
        return __dot2add_i16i16packed_sat(a, b, acc);
    } else {
        int32 tmp1 = (int32)((int16)((a >> 16) & 0xFFFF)) * (int32)((int16)((b >> 16) & 0xFFFF));
        int32 tmp2 = (int32)((int16)(a & 0xFFFF)) * (int32)((int16)(b & 0xFFFF));
        return saturating_add(saturating_add(tmp1, tmp2), acc);
    }
}

// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in a
// with corresponding unsigned 16-bit integers in b, producing 2 intermediate unsigned 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in acc, and return the result.
__declspec(safe) static inline varying uint32 dot2add_u16u16packed(varying uint32 a, varying uint32 b,
                                                                   varying uint32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot2add_u16u16packed(a, b, acc);
    } else {
        // Zero-extend both operands as per the pseudocode (UU version)
        uint32 tmp1 = (uint32)((a >> 16) & 0xFFFF) * (uint32)((b >> 16) & 0xFFFF);
        uint32 tmp2 = (uint32)(a & 0xFFFF) * (uint32)(b & 0xFFFF);
        return acc + tmp1 + tmp2;
    }
}

// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in a
// with corresponding unsigned 16-bit integers in b, producing 2 intermediate unsigned 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in acc using unsigned saturation, and return the result.
__declspec(safe) static inline varying uint32 dot2add_u16u16packed_sat(varying uint32 a, varying uint32 b,
                                                                       varying uint32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot2add_u16u16packed_sat(a, b, acc);
    } else {
        // Zero-extend both operands as per the pseudocode (UU version)
        uint32 tmp1 = (uint32)((a >> 16) & 0xFFFF) * (uint32)((b >> 16) & 0xFFFF);
        uint32 tmp2 = (uint32)(a & 0xFFFF) * (uint32)(b & 0xFFFF);
        // Use unsigned saturation for UU version
        return saturating_add(saturating_add(acc, tmp1), tmp2);
    }
}

// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in a
// with corresponding signed 16-bit integers in b, producing 2 intermediate signed 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in acc, and return the result.
__declspec(safe) static inline varying int32 dot2add_u16i16packed(varying uint32 a, varying uint32 b,
                                                                  varying int32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot2add_u16i16packed(a, b, acc);
    } else {
        // Zero-extend operand a, sign-extend operand b as per pseudocode (US version)
        int32 tmp1 = (int32)((uint16)((a >> 16) & 0xFFFF)) * (int32)((int16)((b >> 16) & 0xFFFF));
        int32 tmp2 = (int32)((uint16)(a & 0xFFFF)) * (int32)((int16)(b & 0xFFFF));
        return acc + tmp1 + tmp2;
    }
}

// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in a
// with corresponding signed 16-bit integers in b, producing 2 intermediate signed 32-bit results.
// Sum these 2 results with the corresponding 32-bit integer in acc using signed saturation, and return the result.
__declspec(safe) static inline varying int32 dot2add_u16i16packed_sat(varying uint32 a, varying uint32 b,
                                                                      varying int32 acc) {
    if (__have_intel_vnni_int8) {
        return __dot2add_u16i16packed_sat(a, b, acc);
    } else {
        // Zero-extend operand a, sign-extend operand b as per pseudocode (US version)
        int32 tmp1 = (int32)((uint16)((a >> 16) & 0xFFFF)) * (int32)((int16)((b >> 16) & 0xFFFF));
        int32 tmp2 = (int32)((uint16)(a & 0xFFFF)) * (int32)((int16)(b & 0xFFFF));
        // Use signed saturation for US version
        return saturating_add(saturating_add(acc, tmp1), tmp2);
    }
}