redgpu/ispc: Intel® Implicit SPMD Program Compiler

stdlib/include/builtins.isph
599 строк42 KB
Arina Neshlyaeva
Added generic implementations of dot4add_u8u8packed and dot4add_i8i8packed
19 дек 2024, 03:25
19 дек 2024, 03:2573de2c1
// -*- mode: c++ -*-
// Copyright (c) 2024, Intel Corporation
// SPDX-License-Identifier: BSD-3-Clause
//
// @file builtins.isph
// @brief This file contains the declarations of the built-in functions that
// are defined in builtins*.ll files but used in the ISPC code of standard
// library. These functions should not be used outside of the standard library
// code.

#pragma once

// Macros for attributes
#define NOESCAPE __attribute__((noescape))
#define ADDRSPACE(N) __attribute__((address_space(N)))
#define READONLY __attribute__((memory("read")))
#define READNONE __attribute__((memory("none")))
#define EXT __attribute__((unmangled)) __attribute__((cdecl)) unmasked
// all ISPC pointer arguments of functions are noalias so we don't need to
// support it until it changed.
// #define NOALIAS __attribute__((noalias))
#define NOALIAS

// trigonometric functions
EXT READNONE uniform double __acos_uniform_double(uniform double);
EXT READNONE uniform float __acos_uniform_float(uniform float);
EXT READNONE uniform float16 __acos_uniform_half(uniform float16);
EXT READNONE varying double __acos_varying_double(varying double);
EXT READNONE varying float __acos_varying_float(varying float);
EXT READNONE varying float16 __acos_varying_half(varying float16);
EXT READNONE uniform double __asin_uniform_double(uniform double);
EXT READNONE uniform float __asin_uniform_float(uniform float);
EXT READNONE uniform float16 __asin_uniform_half(uniform float16);
EXT READNONE varying double __asin_varying_double(varying double);
EXT READNONE varying float __asin_varying_float(varying float);
EXT READNONE varying float16 __asin_varying_half(varying float16);
EXT READNONE uniform double __atan2_uniform_double(uniform double, uniform double);
EXT READNONE uniform float __atan2_uniform_float(uniform float, uniform float);
EXT READNONE uniform float16 __atan2_uniform_half(uniform float16, uniform float16);
EXT READNONE varying double __atan2_varying_double(varying double, varying double);
EXT READNONE varying float __atan2_varying_float(varying float, varying float);
EXT READNONE varying float16 __atan2_varying_half(varying float16, varying float16);
EXT READNONE uniform double __atan_uniform_double(uniform double);
EXT READNONE uniform float __atan_uniform_float(uniform float);
EXT READNONE uniform float16 __atan_uniform_half(uniform float16);
EXT READNONE varying double __atan_varying_double(varying double);
EXT READNONE varying float __atan_varying_float(varying float);
EXT READNONE varying float16 __atan_varying_half(varying float16);
EXT READNONE uniform double __cos_uniform_double(uniform double);
EXT READNONE uniform float __cos_uniform_float(uniform float);
EXT READNONE uniform float16 __cos_uniform_half(uniform float16);
EXT READNONE varying double __cos_varying_double(varying double);
EXT READNONE varying float __cos_varying_float(varying float);
EXT READNONE varying float16 __cos_varying_half(varying float16);
EXT READNONE void __sincos_uniform_double(uniform double, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE void __sincos_uniform_float(uniform float, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE void __sincos_uniform_half(uniform float16, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE void __sincos_varying_double(varying double, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE void __sincos_varying_float(varying float, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE void __sincos_varying_half(varying float16, uniform int8 *uniform, uniform int8 *uniform);
EXT READNONE uniform double __sin_uniform_double(uniform double);
EXT READNONE uniform float __sin_uniform_float(uniform float);
EXT READNONE uniform float16 __sin_uniform_half(uniform float16);
EXT READNONE varying double __sin_varying_double(varying double);
EXT READNONE varying float __sin_varying_float(varying float);
EXT READNONE varying float16 __sin_varying_half(varying float16);
EXT READNONE uniform double __tan_uniform_double(uniform double);
EXT READNONE uniform float __tan_uniform_float(uniform float);
EXT READNONE uniform float16 __tan_uniform_half(uniform float16);
EXT READNONE varying double __tan_varying_double(varying double);
EXT READNONE varying float __tan_varying_float(varying float);
EXT READNONE varying float16 __tan_varying_half(varying float16);

// TODO: do they need to be implemented in LLVM IR?
// AOS/SOA conversion primitives
EXT inline void __aos_to_soa2_double(NOALIAS uniform int8 *uniform, uniform int8 *uniform, uniform int8 *uniform);
EXT inline void __aos_to_soa2_float(NOALIAS uniform int8 *uniform, uniform int8 *uniform, uniform int8 *uniform);
EXT inline void __aos_to_soa3_double(NOALIAS uniform int8 *uniform, uniform int8 *uniform, uniform int8 *uniform,
                                     uniform int8 *uniform);
EXT inline void __aos_to_soa3_float(NOALIAS uniform int8 *uniform, uniform int8 *uniform, uniform int8 *uniform,
                                    uniform int8 *uniform);
EXT inline void __aos_to_soa4_double(NOALIAS uniform int8 *uniform, NOALIAS uniform int8 *uniform,
                                     NOALIAS uniform int8 *uniform, NOALIAS uniform int8 *uniform,
                                     NOALIAS uniform int8 *uniform);
EXT inline void __aos_to_soa4_float(NOALIAS uniform int8 *uniform, NOALIAS uniform int8 *uniform,
                                    NOALIAS uniform int8 *uniform, NOALIAS uniform int8 *uniform,
                                    NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos2_double(varying double, varying double, NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos2_float(varying float, varying float, NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos3_double(varying double, varying double, varying double, NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos3_float(varying float, varying float, varying float, NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos4_double(varying double, varying double, varying double, varying double,
                                     NOALIAS uniform int8 *uniform);
EXT inline void __soa_to_aos4_float(varying float, varying float, varying float, varying float,
                                    NOALIAS uniform int8 *uniform);

// atomics and memory barriers
EXT inline varying int32 __atomic_add_int32_global(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline varying int64 __atomic_add_int64_global(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline uniform int32 __atomic_add_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_add_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline varying int32 __atomic_and_int32_global(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline varying int64 __atomic_and_int64_global(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline uniform int32 __atomic_and_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_and_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline varying double __atomic_compare_exchange_double_global(uniform int8 *uniform, varying double, varying double,
                                                                  UIntMaskType);
EXT inline varying float __atomic_compare_exchange_float_global(uniform int8 *uniform, varying float, varying float,
                                                                UIntMaskType);
EXT inline varying int32 __atomic_compare_exchange_int32_global(uniform int8 *uniform, varying int32, varying int32,
                                                                UIntMaskType);
EXT inline varying int64 __atomic_compare_exchange_int64_global(uniform int8 *uniform, varying int64, varying int64,
                                                                UIntMaskType);
EXT inline uniform double __atomic_compare_exchange_uniform_double_global(uniform int8 *uniform, uniform double,
                                                                          uniform double);
EXT inline uniform float __atomic_compare_exchange_uniform_float_global(uniform int8 *uniform, uniform float,
                                                                        uniform float);
EXT inline uniform int32 __atomic_compare_exchange_uniform_int32_global(uniform int8 *uniform, uniform int32,
                                                                        uniform int32);
EXT inline uniform int64 __atomic_compare_exchange_uniform_int64_global(uniform int8 *uniform, uniform int64,
                                                                        uniform int64);
EXT inline varying double __atomic_fadd_double_global(uniform int8 *uniform, varying double, UIntMaskType);
EXT inline varying float __atomic_fadd_float_global(uniform int8 *uniform, varying float, UIntMaskType);
EXT inline uniform double __atomic_fadd_uniform_double_global(uniform int8 *uniform, uniform double);
EXT inline uniform float __atomic_fadd_uniform_float_global(uniform int8 *uniform, uniform float);
EXT inline varying double __atomic_fmax_double_global(uniform int8 *uniform, varying double, UIntMaskType);
EXT inline varying float __atomic_fmax_float_global(uniform int8 *uniform, varying float, UIntMaskType);
EXT inline uniform double __atomic_fmax_uniform_double_global(uniform int8 *uniform, uniform double);
EXT inline uniform float __atomic_fmax_uniform_float_global(uniform int8 *uniform, uniform float);
EXT inline varying double __atomic_fmin_double_global(uniform int8 *uniform, varying double, UIntMaskType);
EXT inline varying float __atomic_fmin_float_global(uniform int8 *uniform, varying float, UIntMaskType);
EXT inline uniform double __atomic_fmin_uniform_double_global(uniform int8 *uniform, uniform double);
EXT inline uniform float __atomic_fmin_uniform_float_global(uniform int8 *uniform, uniform float);
EXT inline varying double __atomic_fsub_double_global(uniform int8 *uniform, varying double, UIntMaskType);
EXT inline varying float __atomic_fsub_float_global(uniform int8 *uniform, varying float, UIntMaskType);
EXT inline uniform double __atomic_fsub_uniform_double_global(uniform int8 *uniform, uniform double);
EXT inline uniform float __atomic_fsub_uniform_float_global(uniform int8 *uniform, uniform float);
EXT inline uniform int32 __atomic_max_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_max_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline uniform int32 __atomic_min_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_min_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline varying int32 __atomic_or_int32_global(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline varying int64 __atomic_or_int64_global(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline uniform int32 __atomic_or_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_or_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline varying int32 __atomic_sub_int32_global(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline varying int64 __atomic_sub_int64_global(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline uniform int32 __atomic_sub_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_sub_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline uniform double __atomic_swap_uniform_double_global(uniform int8 *uniform, uniform double);
EXT inline uniform float __atomic_swap_uniform_float_global(uniform int8 *uniform, uniform float);
EXT inline uniform int32 __atomic_swap_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_swap_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline uniform uint32 __atomic_umax_uniform_uint32_global(uniform int8 *uniform, uniform uint32);
EXT inline uniform uint64 __atomic_umax_uniform_uint64_global(uniform int8 *uniform, uniform uint64);
EXT inline uniform uint32 __atomic_umin_uniform_uint32_global(uniform int8 *uniform, uniform uint32);
EXT inline uniform uint64 __atomic_umin_uniform_uint64_global(uniform int8 *uniform, uniform uint64);
EXT inline varying int32 __atomic_xor_int32_global(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline varying int64 __atomic_xor_int64_global(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline uniform int32 __atomic_xor_uniform_int32_global(uniform int8 *uniform, uniform int32);
EXT inline uniform int64 __atomic_xor_uniform_int64_global(uniform int8 *uniform, uniform int64);
EXT inline READNONE void __memory_barrier();

EXT uniform int16 __padds_ui16(uniform int16, uniform int16);
EXT uniform int32 __padds_ui32(uniform int32, uniform int32);
EXT uniform int64 __padds_ui64(uniform int64, uniform int64);
EXT uniform int8 __padds_ui8(uniform int8, uniform int8);
EXT varying int16 __padds_vi16(varying int16, varying int16);
EXT varying int32 __padds_vi32(varying int32, varying int32);
EXT varying int64 __padds_vi64(varying int64, varying int64);
EXT varying int8 __padds_vi8(varying int8, varying int8);
EXT uniform uint16 __paddus_ui16(uniform uint16, uniform uint16);
EXT uniform uint32 __paddus_ui32(uniform uint32, uniform uint32);
EXT uniform uint64 __paddus_ui64(uniform uint64, uniform uint64);
EXT uniform uint8 __paddus_ui8(uniform uint8, uniform uint8);
EXT varying uint16 __paddus_vi16(varying uint16, varying uint16);
EXT varying uint32 __paddus_vi32(varying uint32, varying uint32);
EXT varying uint64 __paddus_vi64(varying uint64, varying uint64);
EXT varying uint8 __paddus_vi8(varying uint8, varying uint8);
EXT uniform int16 __pmuls_ui16(uniform int16, uniform int16);
EXT uniform int32 __pmuls_ui32(uniform int32, uniform int32);
EXT uniform int64 __pmuls_ui64(uniform int64, uniform int64);
EXT uniform int8 __pmuls_ui8(uniform int8, uniform int8);
EXT varying int16 __pmuls_vi16(varying int16, varying int16);
EXT varying int32 __pmuls_vi32(varying int32, varying int32);
EXT varying int64 __pmuls_vi64(varying int64, varying int64);
EXT varying int8 __pmuls_vi8(varying int8, varying int8);
EXT uniform uint16 __pmulus_ui16(uniform uint16, uniform uint16);
EXT uniform uint32 __pmulus_ui32(uniform uint32, uniform uint32);
EXT uniform uint64 __pmulus_ui64(uniform uint64, uniform uint64);
EXT uniform uint8 __pmulus_ui8(uniform uint8, uniform uint8);
EXT varying uint16 __pmulus_vi16(varying uint16, varying uint16);
EXT varying uint32 __pmulus_vi32(varying uint32, varying uint32);
EXT varying uint64 __pmulus_vi64(varying uint64, varying uint64);
EXT varying uint8 __pmulus_vi8(varying uint8, varying uint8);
EXT uniform int16 __psubs_ui16(uniform int16, uniform int16);
EXT uniform int32 __psubs_ui32(uniform int32, uniform int32);
EXT uniform int64 __psubs_ui64(uniform int64, uniform int64);
EXT uniform int8 __psubs_ui8(uniform int8, uniform int8);
EXT varying int16 __psubs_vi16(varying int16, varying int16);
EXT varying int32 __psubs_vi32(varying int32, varying int32);
EXT varying int64 __psubs_vi64(varying int64, varying int64);
EXT varying int8 __psubs_vi8(varying int8, varying int8);
EXT uniform uint16 __psubus_ui16(uniform uint16, uniform uint16);
EXT uniform uint32 __psubus_ui32(uniform uint32, uniform uint32);
EXT uniform uint64 __psubus_ui64(uniform uint64, uniform uint64);
EXT uniform uint8 __psubus_ui8(uniform uint8, uniform uint8);
EXT varying uint16 __psubus_vi16(varying uint16, varying uint16);
EXT varying uint32 __psubus_vi32(varying uint32, varying uint32);
EXT varying uint64 __psubus_vi64(varying uint64, varying uint64);
EXT varying uint8 __psubus_vi8(varying uint8, varying uint8);

// float math functions
EXT inline READNONE uniform double __ceil_uniform_double(uniform double);
EXT inline READNONE uniform float __ceil_uniform_float(uniform float);
EXT inline READNONE uniform float16 __ceil_uniform_half(uniform float16);
EXT inline READNONE varying double __ceil_varying_double(varying double);
EXT inline READNONE varying float __ceil_varying_float(varying float);
EXT inline READNONE varying float16 __ceil_varying_half(varying float16);
EXT inline READNONE uniform double __exp_uniform_double(uniform double);
EXT inline READNONE uniform float __exp_uniform_float(uniform float);
EXT inline READNONE uniform float16 __exp_uniform_half(uniform float16);
EXT inline READNONE varying double __exp_varying_double(varying double);
EXT inline READNONE varying float __exp_varying_float(varying float);
EXT inline READNONE varying float16 __exp_varying_half(varying float16);
EXT inline READNONE uniform double __floor_uniform_double(uniform double);
EXT inline READNONE uniform float __floor_uniform_float(uniform float);
EXT inline READNONE uniform float16 __floor_uniform_half(uniform float16);
EXT inline READNONE varying double __floor_varying_double(varying double);
EXT inline READNONE varying float __floor_varying_float(varying float);
EXT inline READNONE varying float16 __floor_varying_half(varying float16);
EXT inline READNONE uniform double __log_uniform_double(uniform double);
EXT inline READNONE uniform float __log_uniform_float(uniform float);
EXT inline READNONE uniform float16 __log_uniform_half(uniform float16);
EXT inline READNONE varying double __log_varying_double(varying double);
EXT inline READNONE varying float __log_varying_float(varying float);
EXT inline READNONE varying float16 __log_varying_half(varying float16);
EXT inline READNONE uniform double __pow_uniform_double(uniform double, uniform double);
EXT inline READNONE uniform float __pow_uniform_float(uniform float, uniform float);
EXT inline READNONE uniform float16 __pow_uniform_half(uniform float16, uniform float16);
EXT inline READNONE varying double __pow_varying_double(varying double, varying double);
EXT inline READNONE varying float __pow_varying_float(varying float, varying float);
EXT inline READNONE varying float16 __pow_varying_half(varying float16, varying float16);
EXT inline READNONE uniform double __rcp_fast_uniform_double(uniform double);
EXT inline READNONE uniform float __rcp_fast_uniform_float(uniform float);
EXT inline READNONE varying double __rcp_fast_varying_double(varying double);
EXT inline READNONE varying float __rcp_fast_varying_float(varying float);
EXT inline READNONE uniform double __rcp_uniform_double(uniform double);
EXT inline READNONE uniform float __rcp_uniform_float(uniform float);
EXT inline READNONE uniform float16 __rcp_uniform_half(uniform float16);
EXT inline READNONE varying double __rcp_varying_double(varying double);
EXT inline READNONE varying float __rcp_varying_float(varying float);
EXT inline READNONE varying float16 __rcp_varying_half(varying float16);
EXT inline READNONE uniform double __round_uniform_double(uniform double);
EXT inline READNONE uniform float __round_uniform_float(uniform float);
EXT inline READNONE uniform float16 __round_uniform_half(uniform float16);
EXT inline READNONE varying double __round_varying_double(varying double);
EXT inline READNONE varying float __round_varying_float(varying float);
EXT inline READNONE varying float16 __round_varying_half(varying float16);
EXT inline READNONE uniform double __rsqrt_fast_uniform_double(uniform double);
EXT inline READNONE uniform float __rsqrt_fast_uniform_float(uniform float);
EXT inline READNONE varying double __rsqrt_fast_varying_double(varying double);
EXT inline READNONE varying float __rsqrt_fast_varying_float(varying float);
EXT inline READNONE uniform double __rsqrt_uniform_double(uniform double);
EXT inline READNONE uniform float __rsqrt_uniform_float(uniform float);
EXT inline READNONE uniform float16 __rsqrt_uniform_half(uniform float16);
EXT inline READNONE varying double __rsqrt_varying_double(varying double);
EXT inline READNONE varying float __rsqrt_varying_float(varying float);
EXT inline READNONE varying float16 __rsqrt_varying_half(varying float16);
EXT inline READNONE uniform double __sqrt_uniform_double(uniform double);
EXT inline READNONE uniform float __sqrt_uniform_float(uniform float);
EXT inline READNONE uniform float16 __sqrt_uniform_half(uniform float16);
EXT inline READNONE varying double __sqrt_varying_double(varying double);
EXT inline READNONE varying float __sqrt_varying_float(varying float);
EXT inline READNONE varying float16 __sqrt_varying_half(varying float16);
EXT inline READNONE uniform double __trunc_uniform_double(uniform double);
EXT inline READNONE uniform float __trunc_uniform_float(uniform float);
EXT inline READNONE uniform float16 __trunc_uniform_half(uniform float16);
EXT inline READNONE varying double __trunc_varying_double(varying double);
EXT inline READNONE varying float __trunc_varying_float(varying float);
EXT inline READNONE varying float16 __trunc_varying_half(varying float16);

EXT uniform int32 __popcnt_int32(uniform int32);
EXT uniform int64 __popcnt_int64(uniform int64);

// read hw clock
EXT uniform int64 __clock();

// prefix sum stuff
EXT inline varying double __exclusive_scan_add_double(varying double, UIntMaskType);
EXT inline varying float __exclusive_scan_add_float(varying float, UIntMaskType);
EXT inline varying float16 __exclusive_scan_add_half(varying float16, UIntMaskType);
EXT inline varying int32 __exclusive_scan_add_i32(varying int32, UIntMaskType);
EXT inline varying int64 __exclusive_scan_add_i64(varying int64, UIntMaskType);
EXT inline varying int32 __exclusive_scan_and_i32(varying int32, UIntMaskType);
EXT inline varying int64 __exclusive_scan_and_i64(varying int64, UIntMaskType);
EXT inline varying int32 __exclusive_scan_or_i32(varying int32, UIntMaskType);
EXT inline varying int64 __exclusive_scan_or_i64(varying int64, UIntMaskType);

// vector ops
EXT inline READNONE uniform bool __extract_bool(IntMaskType, uniform int32);
EXT inline READNONE uniform int16 __extract_int16(varying int16, uniform int32);
EXT inline READNONE uniform int32 __extract_int32(varying int32, uniform int32);
EXT inline READNONE uniform int64 __extract_int64(varying int64, uniform int32);
EXT inline READNONE uniform int8 __extract_int8(varying int8, uniform int32);
EXT inline READNONE IntMaskType __insert_bool(IntMaskType, uniform int32, uniform bool);
EXT inline READNONE varying int16 __insert_int16(varying int16, uniform int32, uniform int16);
EXT inline READNONE varying int32 __insert_int32(varying int32, uniform int32, uniform int32);
EXT inline READNONE varying int64 __insert_int64(varying int64, uniform int32, uniform int64);
EXT inline READNONE varying int8 __insert_int8(varying int8, uniform int32, uniform int8);

// dot product
EXT READNONE varying int32 __dot2add_i16packed_sat(varying uint32, varying uint32, varying int32);
EXT READNONE varying int32 __dot2add_i16packed(varying uint32, varying uint32, varying int32);
EXT READNONE varying int32 __dot4add_u8i8packed_sat(varying uint32, varying uint32, varying int32);
EXT READNONE varying int32 __dot4add_u8i8packed(varying uint32, varying uint32, varying int32);
EXT READNONE varying uint32 __dot4add_u8u8packed_sat(varying uint32, varying uint32, varying uint32);
EXT READNONE varying uint32 __dot4add_u8u8packed(varying uint32, varying uint32, varying uint32);
EXT READNONE varying int32 __dot4add_i8i8packed_sat(varying uint32, varying uint32, varying int32);
EXT READNONE varying int32 __dot4add_i8i8packed(varying uint32, varying uint32, varying int32);

// various bitcasts from one type to another
EXT inline READNONE uniform double __doublebits_uniform_int64(uniform int64);
EXT inline READNONE varying double __doublebits_varying_int64(varying int64);
EXT inline READNONE uniform float __floatbits_uniform_int32(uniform int32);
EXT inline READNONE varying float __floatbits_varying_int32(varying int32);
EXT uniform int16 __float_to_half_uniform(uniform float);
EXT varying int16 __float_to_half_varying(varying float);
EXT inline READNONE uniform float16 __halfbits_uniform_int16(uniform int16);
EXT inline READNONE varying float16 __halfbits_varying_int16(varying int16);
EXT uniform float __half_to_float_uniform(uniform int16);
EXT varying float __half_to_float_varying(varying int16);
EXT inline READNONE uniform int64 __intbits_uniform_double(uniform double);
EXT inline READNONE uniform int32 __intbits_uniform_float(uniform float);
EXT inline READNONE uniform int16 __intbits_uniform_half(uniform float16);
EXT inline READNONE varying int64 __intbits_varying_double(varying double);
EXT inline READNONE varying int32 __intbits_varying_float(varying float);
EXT inline READNONE varying int16 __intbits_varying_half(varying float16);
EXT uniform float __undef_uniform();
EXT varying float __undef_varying();

// fast math function
EXT inline void __fastmath();
EXT void __scatter_elt32_double(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying double,
                                uniform int32);
EXT void __scatter_elt32_float(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying float,
                               uniform int32);
EXT void __scatter_elt32_half(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying float16,
                              uniform int32);
EXT void __scatter_elt32_i16(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying int16,
                             uniform int32);
EXT void __scatter_elt32_i32(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying int32,
                             uniform int32);
EXT void __scatter_elt32_i64(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying int64,
                             uniform int32);
EXT void __scatter_elt32_i8(uniform int8 *uniform, varying int32, uniform int32, varying int32, varying int8,
                            uniform int32);
EXT void __scatter_elt64_double(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying double,
                                uniform int32);
EXT void __scatter_elt64_float(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying float,
                               uniform int32);
EXT void __scatter_elt64_half(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying float16,
                              uniform int32);
EXT void __scatter_elt64_i16(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying int16,
                             uniform int32);
EXT void __scatter_elt64_i32(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying int32,
                             uniform int32);
EXT void __scatter_elt64_i64(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying int64,
                             uniform int32);
EXT void __scatter_elt64_i8(uniform int8 *uniform, varying int64, uniform int32, varying int64, varying int8,
                            uniform int32);

// TODO: are these used at all?
EXT READNONE varying int16 __idiv_int16(varying int16, varying int16);
EXT READNONE varying int32 __idiv_int32(varying int32, varying int32);
EXT READNONE varying int8 __idiv_int8(varying int8, varying int8);
EXT READNONE varying int16 __idiv_uint16(varying int16, varying int16);
EXT READNONE varying int32 __idiv_uint32(varying int32, varying int32);
EXT READNONE varying int8 __idiv_uint8(varying int8, varying int8);
EXT varying double __masked_load_blend_double(uniform int8 *uniform, UIntMaskType);
EXT varying float __masked_load_blend_float(uniform int8 *uniform, UIntMaskType);
EXT varying float16 __masked_load_blend_half(uniform int8 *uniform, UIntMaskType);
EXT varying int16 __masked_load_blend_i16(uniform int8 *uniform, UIntMaskType);
EXT varying int32 __masked_load_blend_i32(uniform int8 *uniform, UIntMaskType);
EXT varying int64 __masked_load_blend_i64(uniform int8 *uniform, UIntMaskType);
EXT varying int8 __masked_load_blend_i8(uniform int8 *uniform, UIntMaskType);
EXT varying double __masked_load_private_double(uniform int8 *uniform, UIntMaskType);
EXT varying float __masked_load_private_float(uniform int8 *uniform, UIntMaskType);
EXT varying int16 __masked_load_private_i16(uniform int8 *uniform, UIntMaskType);
EXT varying int32 __masked_load_private_i32(uniform int8 *uniform, UIntMaskType);
EXT varying int64 __masked_load_private_i64(uniform int8 *uniform, UIntMaskType);
EXT varying int8 __masked_load_private_i8(uniform int8 *uniform, UIntMaskType);

// TODO: fix signdness
// 64-bit integer min and max functions
EXT inline READNONE uniform double __max_uniform_double(uniform double, uniform double);
EXT inline READNONE uniform float __max_uniform_float(uniform float, uniform float);
EXT inline READNONE uniform float16 __max_uniform_half(uniform float16, uniform float16);
EXT inline READNONE uniform int32 __max_uniform_int32(uniform int32, uniform int32);
EXT inline READNONE uniform int64 __max_uniform_int64(uniform int64, uniform int64);
EXT inline READNONE uniform uint32 __max_uniform_uint32(uniform uint32, uniform uint32);
EXT inline READNONE uniform uint64 __max_uniform_uint64(uniform uint64, uniform uint64);
EXT inline READNONE varying double __max_varying_double(varying double, varying double);
EXT inline READNONE varying float __max_varying_float(varying float, varying float);
EXT inline READNONE varying float16 __max_varying_half(varying float16, varying float16);
EXT inline READNONE varying int32 __max_varying_int32(varying int32, varying int32);
EXT inline READNONE varying int64 __max_varying_int64(varying int64, varying int64);
EXT inline READNONE varying uint32 __max_varying_uint32(varying uint32, varying uint32);
EXT inline READNONE varying uint64 __max_varying_uint64(varying uint64, varying uint64);
EXT inline READNONE uniform double __min_uniform_double(uniform double, uniform double);
EXT inline READNONE uniform float __min_uniform_float(uniform float, uniform float);
EXT inline READNONE uniform float16 __min_uniform_half(uniform float16, uniform float16);
EXT inline READNONE uniform int32 __min_uniform_int32(uniform int32, uniform int32);
EXT inline READNONE uniform int64 __min_uniform_int64(uniform int64, uniform int64);
EXT inline READNONE uniform uint32 __min_uniform_uint32(uniform uint32, uniform uint32);
EXT inline READNONE uniform uint64 __min_uniform_uint64(uniform uint64, uniform uint64);
EXT inline READNONE varying double __min_varying_double(varying double, varying double);
EXT inline READNONE varying float __min_varying_float(varying float, varying float);
EXT inline READNONE varying float16 __min_varying_half(varying float16, varying float16);
EXT inline READNONE varying int32 __min_varying_int32(varying int32, varying int32);
EXT inline READNONE varying int64 __min_varying_int64(varying int64, varying int64);
EXT inline READNONE varying uint32 __min_varying_uint32(varying uint32, varying uint32);
EXT inline READNONE varying uint64 __min_varying_uint64(varying uint64, varying uint64);

// TODO: rewrite using --enable-llvm-intrinsics in stdlib.ispc
// memcpy/memmove/memset
EXT inline void __memcpy32(uniform int8 *uniform, uniform int8 *uniform, uniform int32);
EXT inline void __memcpy64(uniform int8 *uniform, uniform int8 *uniform, uniform int64);
EXT inline void __memmove32(uniform int8 *uniform, uniform int8 *uniform, uniform int32);
EXT inline void __memmove64(uniform int8 *uniform, uniform int8 *uniform, uniform int64);
EXT inline void __memset32(uniform int8 *uniform, uniform int8, uniform int32);
EXT inline void __memset64(uniform int8 *uniform, uniform int8, uniform int64);

// TODO: check types with sign version of functions
// reduction functions
EXT inline READNONE uniform double __reduce_add_double(varying double);
EXT inline READNONE uniform float __reduce_add_float(varying float);
EXT inline READNONE uniform float16 __reduce_add_half(varying float16);
EXT inline READNONE uniform int16 __reduce_add_int16(varying int16);
EXT inline READNONE uniform int32 __reduce_add_int32(varying int32);
EXT inline READNONE uniform int64 __reduce_add_int64(varying int64);
EXT inline READNONE uniform int16 __reduce_add_int8(varying int8);
EXT inline uniform bool __reduce_equal_double(varying double, uniform int8 *uniform, UIntMaskType);
EXT inline uniform bool __reduce_equal_float(varying float, uniform int8 *uniform, UIntMaskType);
EXT inline uniform bool __reduce_equal_half(varying float16, uniform int8 *uniform, UIntMaskType);
EXT inline uniform bool __reduce_equal_int32(varying int32, uniform int8 *uniform, UIntMaskType);
EXT inline uniform bool __reduce_equal_int64(varying int64, uniform int8 *uniform, UIntMaskType);
EXT inline READNONE uniform double __reduce_max_double(varying double);
EXT inline READNONE uniform float __reduce_max_float(varying float);
EXT inline READNONE uniform float16 __reduce_max_half(varying float16);
EXT inline READNONE uniform int32 __reduce_max_int32(varying int32);
EXT inline READNONE uniform int64 __reduce_max_int64(varying int64);
EXT inline READNONE uniform uint32 __reduce_max_uint32(varying uint32);
EXT inline READNONE uniform uint64 __reduce_max_uint64(varying uint64);
EXT inline READNONE uniform double __reduce_min_double(varying double);
EXT inline READNONE uniform float __reduce_min_float(varying float);
EXT inline READNONE uniform float16 __reduce_min_half(varying float16);
EXT inline READNONE uniform int32 __reduce_min_int32(varying int32);
EXT inline READNONE uniform int64 __reduce_min_int64(varying int64);
EXT inline READNONE uniform uint32 __reduce_min_uint32(varying uint32);
EXT inline READNONE uniform uint64 __reduce_min_uint64(varying uint64);

// TODO: move implementation to stdlib.ispc
// rdrand
EXT uniform bool __rdrand_i16(uniform int8 *uniform);
EXT uniform bool __rdrand_i32(uniform int8 *uniform);
EXT uniform bool __rdrand_i64(uniform int8 *uniform);

// packed load and store helper
#if (ISPC_MASK_BITS == 1)
#define PackedStoreResultType uniform int32
#else
#define PackedStoreResultType UniformMaskType
#endif
EXT inline uniform int32 __packed_load_activei32(uniform int8 *uniform, uniform int8 *uniform, UIntMaskType);
EXT inline uniform int32 __packed_load_activei64(uniform int8 *uniform, uniform int8 *uniform, UIntMaskType);
EXT inline PackedStoreResultType __packed_store_active2i32(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline PackedStoreResultType __packed_store_active2i64(uniform int8 *uniform, varying int64, UIntMaskType);
EXT inline PackedStoreResultType __packed_store_activei32(uniform int8 *uniform, varying int32, UIntMaskType);
EXT inline PackedStoreResultType __packed_store_activei64(uniform int8 *uniform, varying int64, UIntMaskType);

// TODO: rewrite in stdlib.ispc
EXT void __prefetch_read_sized_uniform_1(uniform int8 *uniform, uniform int8);
EXT void __prefetch_read_sized_uniform_2(uniform int8 *uniform, uniform int8);
EXT void __prefetch_read_sized_uniform_3(uniform int8 *uniform, uniform int8);
EXT void __prefetch_read_sized_uniform_nt(uniform int8 *uniform, uniform int8);
EXT void __prefetch_read_sized_varying_1(varying int64, uniform int8, UIntMaskType);
EXT void __prefetch_read_sized_varying_2(varying int64, uniform int8, UIntMaskType);
EXT void __prefetch_read_sized_varying_3(varying int64, uniform int8, UIntMaskType);
EXT void __prefetch_read_sized_varying_nt(varying int64, uniform int8, UIntMaskType);
EXT void __prefetch_read_uniform_1(uniform int8 *uniform);
EXT void __prefetch_read_uniform_2(uniform int8 *uniform);
EXT void __prefetch_read_uniform_3(uniform int8 *uniform);
EXT void __prefetch_read_uniform_nt(uniform int8 *uniform);
EXT void __prefetch_write_uniform_1(uniform int8 *uniform);
EXT void __prefetch_write_uniform_2(uniform int8 *uniform);
EXT void __prefetch_write_uniform_3(uniform int8 *uniform);

// sign extension
EXT inline READNONE uniform int32 __sext_uniform_bool(uniform bool);
EXT inline READNONE varying int32 __sext_varying_bool(bool);

// streaming stores
EXT inline uniform double __streaming_load_uniform_double(NOESCAPE uniform int8 *uniform);
EXT inline uniform float __streaming_load_uniform_float(NOESCAPE uniform int8 *uniform);
EXT inline uniform float16 __streaming_load_uniform_half(NOESCAPE uniform int8 *uniform);
EXT inline uniform int16 __streaming_load_uniform_i16(NOESCAPE uniform int8 *uniform);
EXT inline uniform int32 __streaming_load_uniform_i32(NOESCAPE uniform int8 *uniform);
EXT inline uniform int64 __streaming_load_uniform_i64(NOESCAPE uniform int8 *uniform);
EXT inline uniform int8 __streaming_load_uniform_i8(NOESCAPE uniform int8 *uniform);
EXT inline varying double __streaming_load_varying_double(NOESCAPE uniform int8 *uniform);
EXT inline varying float __streaming_load_varying_float(NOESCAPE uniform int8 *uniform);
EXT inline varying float16 __streaming_load_varying_half(NOESCAPE uniform int8 *uniform);
EXT inline varying int16 __streaming_load_varying_i16(NOESCAPE uniform int8 *uniform);
EXT inline varying int32 __streaming_load_varying_i32(NOESCAPE uniform int8 *uniform);
EXT inline varying int64 __streaming_load_varying_i64(NOESCAPE uniform int8 *uniform);
EXT inline varying int8 __streaming_load_varying_i8(NOESCAPE uniform int8 *uniform);
EXT inline void __streaming_store_uniform_double(NOESCAPE uniform int8 *uniform, uniform double);
EXT inline void __streaming_store_uniform_float(NOESCAPE uniform int8 *uniform, uniform float);
EXT inline void __streaming_store_uniform_half(NOESCAPE uniform int8 *uniform, uniform float16);
EXT inline void __streaming_store_uniform_i16(NOESCAPE uniform int8 *uniform, uniform int16);
EXT inline void __streaming_store_uniform_i32(NOESCAPE uniform int8 *uniform, uniform int32);
EXT inline void __streaming_store_uniform_i64(NOESCAPE uniform int8 *uniform, uniform int64);
EXT inline void __streaming_store_uniform_i8(NOESCAPE uniform int8 *uniform, uniform int8);
EXT inline void __streaming_store_varying_double(NOESCAPE uniform int8 *uniform, varying double);
EXT inline void __streaming_store_varying_float(NOESCAPE uniform int8 *uniform, varying float);
EXT inline void __streaming_store_varying_half(NOESCAPE uniform int8 *uniform, varying float16);
EXT inline void __streaming_store_varying_i16(NOESCAPE uniform int8 *uniform, varying int16);
EXT inline void __streaming_store_varying_i32(NOESCAPE uniform int8 *uniform, varying int32);
EXT inline void __streaming_store_varying_i64(NOESCAPE uniform int8 *uniform, varying int64);
EXT inline void __streaming_store_varying_i8(NOESCAPE uniform int8 *uniform, varying int8);

// TODO: looks redundant, may call stdlib functions directly from stdlib.ispc
// stdlib transcendentals
EXT inline READNONE uniform float __stdlib_acosf(uniform float);
EXT inline READNONE uniform float __stdlib_asinf(uniform float);
EXT inline READNONE uniform double __stdlib_asin(uniform double);
EXT inline READNONE uniform float __stdlib_atan2f(uniform float, uniform float);
EXT inline READNONE uniform double __stdlib_atan2(uniform double, uniform double);
EXT inline READNONE uniform float __stdlib_atanf(uniform float);
EXT inline READNONE uniform double __stdlib_atan(uniform double);
EXT inline READNONE uniform float __stdlib_cosf(uniform float);
EXT inline READNONE uniform double __stdlib_cos(uniform double);
EXT inline READNONE uniform float __stdlib_expf(uniform float);
EXT inline READNONE uniform double __stdlib_exp(uniform double);
EXT inline READNONE uniform float __stdlib_logf(uniform float);
EXT inline READNONE uniform double __stdlib_log(uniform double);
EXT inline READNONE uniform float __stdlib_powf(uniform float, uniform float);
EXT inline READNONE uniform double __stdlib_pow(uniform double, uniform double);
EXT inline READNONE void __stdlib_sincosf(uniform float, uniform int8 *uniform, uniform int8 *uniform);
EXT inline READNONE void __stdlib_sincos(uniform double, uniform int8 *uniform, uniform int8 *uniform);
EXT inline READNONE uniform float __stdlib_sinf(uniform float);
EXT inline READNONE uniform double __stdlib_sin(uniform double);
EXT inline READNONE uniform float __stdlib_tanf(uniform float);
EXT inline READNONE uniform double __stdlib_tan(uniform double);

EXT inline READNONE varying double __broadcast_double(varying double, uniform int32);
EXT inline READNONE varying float __broadcast_float(varying float, uniform int32);
EXT inline READNONE varying float16 __broadcast_half(varying float16, uniform int32);
EXT inline READNONE varying int16 __broadcast_i16(varying int16, uniform int32);
EXT inline READNONE varying int32 __broadcast_i32(varying int32, uniform int32);
EXT inline READNONE varying int64 __broadcast_i64(varying int64, uniform int32);
EXT inline READNONE varying int8 __broadcast_i8(varying int8, uniform int32);
EXT inline READNONE uniform uint64 __cast_mask_to_i64(UIntMaskType);
EXT inline READNONE varying double __rotate_double(varying double, uniform int32);
EXT inline READNONE varying float __rotate_float(varying float, uniform int32);
EXT inline READNONE varying float16 __rotate_half(varying float16, uniform int32);
EXT inline READNONE varying int16 __rotate_i16(varying int16, uniform int32);
EXT inline READNONE varying int32 __rotate_i32(varying int32, uniform int32);
EXT inline READNONE varying int64 __rotate_i64(varying int64, uniform int32);
EXT inline READNONE varying int8 __rotate_i8(varying int8, uniform int32);
EXT inline READNONE varying double __shift_double(varying double, uniform int32);
EXT inline READNONE varying float __shift_float(varying float, uniform int32);
EXT inline READNONE varying float16 __shift_half(varying float16, uniform int32);
EXT inline READNONE varying int16 __shift_i16(varying int16, uniform int32);
EXT inline READNONE varying int32 __shift_i32(varying int32, uniform int32);
EXT inline READNONE varying int64 __shift_i64(varying int64, uniform int32);
EXT inline READNONE varying int8 __shift_i8(varying int8, uniform int32);
EXT inline READNONE varying double __shuffle2_double(varying double, varying double, varying int32);
EXT inline READNONE varying float __shuffle2_float(varying float, varying float, varying int32);
EXT inline READNONE varying float16 __shuffle2_half(varying float16, varying float16, varying int32);
EXT inline READNONE varying int16 __shuffle2_i16(varying int16, varying int16, varying int32);
EXT inline READNONE varying int32 __shuffle2_i32(varying int32, varying int32, varying int32);
EXT inline READNONE varying int64 __shuffle2_i64(varying int64, varying int64, varying int32);
EXT inline READNONE varying int8 __shuffle2_i8(varying int8, varying int8, varying int32);
EXT inline READNONE varying double __shuffle_double(varying double, varying int32);
EXT inline READNONE varying float __shuffle_float(varying float, varying int32);
EXT inline READNONE varying float16 __shuffle_half(varying float16, varying int32);
EXT inline READNONE varying int16 __shuffle_i16(varying int16, varying int32);
EXT inline READNONE varying int32 __shuffle_i32(varying int32, varying int32);
EXT inline READNONE varying int64 __shuffle_i64(varying int64, varying int32);
EXT inline READNONE varying int8 __shuffle_i8(varying int8, varying int32);

#undef NOESCAPE
#undef ADDRSPACE
#undef READONLY
#undef READNONE
#undef NOALIAS
#undef EXT