/
redgpu
/
ispc
Обзор
Документация
Войти
/
redgpu
/
ispc
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
v1.26.0
stdlib/include/stdlib.isph
1 143 строки
67 KB
Arina Neshlyaeva
Added generic implementations of dot4add_u8u8packed and dot4add_i8i8packed
19 дек 2024, 03:25
19 дек 2024, 03:25
73de2c1
Код
Авторство
О чём код?
// -*- mode: c++ -*- // Copyright (c) 2024, Intel Corporation // SPDX-License-Identifier: BSD-3-Clause // @file stdlib.isph // @brief Portion of the ISPC standard library declarations // that are implementated in ISPC. // This whole file is included for every user code compilation unless // --nostdlib is provided. #pragma once /////////////////////////////////////////////////////////////////////////// // Low level primitives __declspec(safe, cost0) inline float16 float16bits(unsigned int16 a); __declspec(safe, cost0) inline uniform float16 float16bits(uniform unsigned int16 a); __declspec(safe, cost0) inline float16 float16bits(int16 a); __declspec(safe, cost0) inline uniform float16 float16bits(uniform int16 a); __declspec(safe, cost0) inline float floatbits(unsigned int a); __declspec(safe, cost0) inline uniform float floatbits(uniform unsigned int a); __declspec(safe, cost0) inline float floatbits(int a); __declspec(safe, cost0) inline uniform float floatbits(uniform int a); __declspec(safe, cost0) inline double doublebits(unsigned int64 a); __declspec(safe, cost0) inline uniform double doublebits(uniform unsigned int64 a); __declspec(safe, cost0) inline unsigned int16 intbits(float16 a); __declspec(safe, cost0) inline uniform unsigned int16 intbits(uniform float16 a); __declspec(safe, cost0) inline unsigned int intbits(float a); __declspec(safe, cost0) inline uniform unsigned int intbits(uniform float a); __declspec(safe, cost0) inline unsigned int64 intbits(double d); __declspec(safe, cost0) inline uniform unsigned int64 intbits(uniform double d); __declspec(safe) inline float broadcast(float v, uniform int i); __declspec(safe) inline int8 broadcast(int8 v, uniform int i); __declspec(safe) inline int16 broadcast(int16 v, uniform int i); __declspec(safe) inline float16 broadcast(float16 v, uniform int i); __declspec(safe) inline int32 broadcast(int32 v, uniform int i); __declspec(safe) inline double broadcast(double v, uniform int i); __declspec(safe) inline int64 broadcast(int64 v, uniform int i); __declspec(safe) inline float rotate(float v, uniform int i); __declspec(safe) inline int8 rotate(int8 v, uniform int i); __declspec(safe) inline int16 rotate(int16 v, uniform int i); __declspec(safe) inline float16 rotate(float16 v, uniform int i); __declspec(safe) inline int32 rotate(int32 v, uniform int i); __declspec(safe) inline double rotate(double v, uniform int i); __declspec(safe) inline int64 rotate(int64 v, uniform int i); __declspec(safe) inline float shift(float v, uniform int i); __declspec(safe) inline int8 shift(int8 v, uniform int i); __declspec(safe) inline int16 shift(int16 v, uniform int i); __declspec(safe) inline float16 shift(float16 v, uniform int i); __declspec(safe) inline int32 shift(int32 v, uniform int i); __declspec(safe) inline double shift(double v, uniform int i); __declspec(safe) inline int64 shift(int64 v, uniform int i); __declspec(safe) inline float shuffle(float v, int i); __declspec(safe) inline int8 shuffle(int8 v, int i); __declspec(safe) inline int16 shuffle(int16 v, int i); __declspec(safe) inline float16 shuffle(float16 v, int i); __declspec(safe) inline int32 shuffle(int32 v, int i); __declspec(safe) inline double shuffle(double v, int i); __declspec(safe) inline int64 shuffle(int64 v, int i); __declspec(safe) inline float shuffle(float v0, float v1, int i); __declspec(safe) inline int8 shuffle(int8 v0, int8 v1, int i); __declspec(safe) inline int16 shuffle(int16 v0, int16 v1, int i); __declspec(safe) inline float16 shuffle(float16 v0, float16 v1, int i); __declspec(safe) inline int32 shuffle(int32 v0, int32 v1, int i); __declspec(safe) inline double shuffle(double v0, double v1, int i); __declspec(safe) inline int64 shuffle(int64 v0, int64 v1, int i); // x[i] __declspec(safe, cost1) inline uniform float extract(float x, uniform int i); __declspec(safe, cost1) inline uniform bool extract(bool x, uniform int i); __declspec(safe, cost1) inline uniform int8 extract(int8 x, uniform int i); __declspec(safe, cost1) inline uniform unsigned int8 extract(unsigned int8 x, uniform int i); __declspec(safe, cost1) inline uniform int16 extract(int16 x, uniform int i); __declspec(safe, cost1) inline uniform unsigned int16 extract(unsigned int16 x, uniform int i); __declspec(safe, cost1) inline uniform float16 extract(float16 x, uniform int i); __declspec(safe, cost1) inline uniform int32 extract(int32 x, uniform int i); __declspec(safe, cost1) inline uniform unsigned int32 extract(unsigned int32 x, uniform int i); __declspec(safe, cost1) inline uniform double extract(double x, uniform int i); __declspec(safe, cost1) inline uniform int64 extract(int64 x, uniform int i); __declspec(safe, cost1) inline uniform unsigned int64 extract(unsigned int64 x, uniform int i); // x[i] = v __declspec(safe, cost1) inline float insert(float x, uniform int i, uniform float v); __declspec(safe, cost1) inline bool insert(bool x, uniform int i, uniform bool v); __declspec(safe, cost1) inline int8 insert(int8 x, uniform int i, uniform int8 v); __declspec(safe, cost1) inline unsigned int8 insert(unsigned int8 x, uniform int i, uniform unsigned int8 v); __declspec(safe, cost1) inline float16 insert(float16 x, uniform int i, uniform float16 v); __declspec(safe, cost1) inline int16 insert(int16 x, uniform int i, uniform int16 v); __declspec(safe, cost1) inline unsigned int16 insert(unsigned int16 x, uniform int i, uniform unsigned int16 v); __declspec(safe, cost1) inline int32 insert(int32 x, uniform int i, uniform int32 v); __declspec(safe, cost1) inline unsigned int32 insert(unsigned int32 x, uniform int i, uniform unsigned int32 v); __declspec(safe, cost1) inline double insert(double x, uniform int i, uniform double v); __declspec(safe, cost1) inline int64 insert(int64 x, uniform int i, uniform int64 v); __declspec(safe, cost1) inline unsigned int64 insert(unsigned int64 x, uniform int i, uniform unsigned int64 v); __declspec(safe, cost1) inline uniform int32 sign_extend(uniform bool v); __declspec(safe, cost1) inline int32 sign_extend(bool v); __declspec(safe) inline uniform bool any(bool v); __declspec(safe) inline uniform bool all(bool v); __declspec(safe) inline uniform bool none(bool v); __declspec(safe) inline uniform int32 popcnt(uniform int32 v); __declspec(safe) inline uniform int popcnt(uniform int64 v); __declspec(safe) inline int popcnt(int v); __declspec(safe) inline int popcnt(int64 v); __declspec(safe) inline uniform int popcnt(bool v); __declspec(safe) inline uniform unsigned int64 lanemask(); __declspec(safe) inline uniform unsigned int64 packmask(bool v); /////////////////////////////////////////////////////////////////////////// // memcpy/memmove/memset inline void memcpy(void *uniform dst, void *uniform src, uniform int32 count); inline void memcpy64(void *uniform dst, void *uniform src, uniform int64 count); inline void memcpy(void *varying dst, void *varying src, int32 count); inline void memcpy64(void *varying dst, void *varying src, int64 count); inline void memmove(void *uniform dst, void *uniform src, uniform int32 count); inline void memmove64(void *uniform dst, void *uniform src, uniform int64 count); inline void memmove(void *varying dst, void *varying src, int32 count); inline void memmove64(void *varying dst, void *varying src, int64 count); inline void memset(void *uniform ptr, uniform int8 val, uniform int32 count); inline void memset64(void *uniform ptr, uniform int8 val, uniform int64 count); inline void memset(void *varying ptr, int8 val, int32 count); inline void memset64(void *varying ptr, int8 val, int64 count); /////////////////////////////////////////////////////////////////////////// // count leading/trailing zeros __declspec(safe, cost1) inline uniform unsigned int32 count_leading_zeros(uniform unsigned int32 v); __declspec(safe, cost1) inline uniform unsigned int64 count_leading_zeros(uniform unsigned int64 v); __declspec(safe, cost1) inline uniform unsigned int32 count_trailing_zeros(uniform unsigned int32 v); __declspec(safe, cost1) inline uniform unsigned int64 count_trailing_zeros(uniform unsigned int64 v); __declspec(safe, cost1) inline uniform int32 count_leading_zeros(uniform int32 v); __declspec(safe, cost1) inline uniform int64 count_leading_zeros(uniform int64 v); __declspec(safe, cost1) inline uniform int32 count_trailing_zeros(uniform int32 v); __declspec(safe, cost1) inline uniform int64 count_trailing_zeros(uniform int64 v); __declspec(safe) inline unsigned int32 count_leading_zeros(unsigned int32 v); __declspec(safe) inline unsigned int64 count_leading_zeros(unsigned int64 v); __declspec(safe) inline unsigned int32 count_trailing_zeros(unsigned int32 v); __declspec(safe) inline unsigned int64 count_trailing_zeros(unsigned int64 v); __declspec(safe) inline int32 count_leading_zeros(int32 v); __declspec(safe) inline int64 count_leading_zeros(int64 v); __declspec(safe) inline int32 count_trailing_zeros(int32 v); __declspec(safe) inline int64 count_trailing_zeros(int64 v); /////////////////////////////////////////////////////////////////////////// // AOS/SOA conversion inline void aos_to_soa2(uniform float a[], varying float *uniform v0, varying float *uniform v1); inline void soa_to_aos2(float v0, float v1, uniform float a[]); inline void aos_to_soa3(uniform float a[], varying float *uniform v0, varying float *uniform v1, varying float *uniform v2); inline void soa_to_aos3(float v0, float v1, float v2, uniform float a[]); inline void aos_to_soa4(uniform float a[], varying float *uniform v0, varying float *uniform v1, varying float *uniform v2, varying float *uniform v3); inline void soa_to_aos4(float v0, float v1, float v2, float v3, uniform float a[]); inline void aos_to_soa2(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1); inline void soa_to_aos2(int32 v0, int32 v1, uniform int32 a[]); inline void aos_to_soa3(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1, varying int32 *uniform v2); inline void soa_to_aos3(int32 v0, int32 v1, int32 v2, uniform int32 a[]); inline void aos_to_soa4(uniform int32 a[], varying int32 *uniform v0, varying int32 *uniform v1, varying int32 *uniform v2, varying int32 *uniform v3); inline void soa_to_aos4(int32 v0, int32 v1, int32 v2, int32 v3, uniform int32 a[]); inline void aos_to_soa2(uniform double a[], varying double *uniform v0, varying double *uniform v1); inline void soa_to_aos2(double v0, double v1, uniform double a[]); inline void aos_to_soa3(uniform double a[], varying double *uniform v0, varying double *uniform v1, varying double *uniform v2); inline void soa_to_aos3(double v0, double v1, double v2, uniform double a[]); inline void aos_to_soa4(uniform double a[], varying double *uniform v0, varying double *uniform v1, varying double *uniform v2, varying double *uniform v3); inline void soa_to_aos4(double v0, double v1, double v2, double v3, uniform double a[]); inline void aos_to_soa2(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1); inline void soa_to_aos2(int64 v0, int64 v1, uniform int64 a[]); inline void aos_to_soa3(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1, varying int64 *uniform v2); inline void soa_to_aos3(int64 v0, int64 v1, int64 v2, uniform int64 a[]); inline void aos_to_soa4(uniform int64 a[], varying int64 *uniform v0, varying int64 *uniform v1, varying int64 *uniform v2, varying int64 *uniform v3); inline void soa_to_aos4(int64 v0, int64 v1, int64 v2, int64 v3, uniform int64 a[]); /////////////////////////////////////////////////////////////////////////// // Prefetching __declspec(safe, cost1) inline void prefetch_l1(const void *uniform ptr); __declspec(safe, cost1) inline void prefetch_l1(const void *uniform ptr, uniform int8 size); __declspec(safe, cost1) inline void prefetch_l2(const void *uniform ptr); __declspec(safe, cost1) inline void prefetch_l2(const void *uniform ptr, uniform int8 size); __declspec(safe, cost1) inline void prefetch_l3(const void *uniform ptr); __declspec(safe, cost1) inline void prefetch_l3(const void *uniform ptr, uniform int8 size); __declspec(safe, cost1) inline void prefetch_nt(const void *uniform ptr); __declspec(safe, cost1) inline void prefetch_nt(const void *uniform ptr, uniform int8 size); __declspec(safe, cost1) inline void prefetchw_l1(const void *uniform ptr); __declspec(safe, cost1) inline void prefetchw_l2(const void *uniform ptr); __declspec(safe, cost1) inline void prefetchw_l3(const void *uniform ptr); inline void prefetch_l1(const void *varying ptr); inline void prefetch_l1(const void *varying ptr, uniform int8 size); inline void prefetch_l2(const void *varying ptr); inline void prefetch_l2(const void *varying ptr, uniform int8 size); inline void prefetch_l3(const void *varying ptr); inline void prefetch_l3(const void *varying ptr, uniform int8 size); inline void prefetch_nt(const void *varying ptr); inline void prefetch_nt(const void *varying ptr, uniform int8 size); __declspec(safe, cost1) inline void prefetchw_l1(const void *varying ptr); __declspec(safe, cost1) inline void prefetchw_l2(const void *varying ptr); __declspec(safe, cost1) inline void prefetchw_l3(const void *varying ptr); /////////////////////////////////////////////////////////////////////////// // non-short-circuiting alternatives __declspec(safe, cost1) inline bool and (bool a, bool b); __declspec(safe, cost1) inline uniform bool and (uniform bool a, uniform bool b); __declspec(safe, cost1) inline bool or (bool a, bool b); __declspec(safe, cost1) inline uniform bool or (uniform bool a, uniform bool b); __declspec(safe, cost1) inline int8 select(bool cond, int8 t, int8 f); __declspec(safe, cost1) inline int8 select(uniform bool cond, int8 t, int8 f); __declspec(safe, cost1) inline uniform int8 select(uniform bool cond, uniform int8 t, uniform int8 f); __declspec(safe, cost1) inline int16 select(bool cond, int16 t, int16 f); __declspec(safe, cost1) inline int16 select(uniform bool cond, int16 t, int16 f); __declspec(safe, cost1) inline uniform int16 select(uniform bool cond, uniform int16 t, uniform int16 f); __declspec(safe, cost1) inline float16 select(bool cond, float16 t, float16 f); __declspec(safe, cost1) inline float16 select(uniform bool cond, float16 t, float16 f); __declspec(safe, cost1) inline uniform float16 select(uniform bool cond, uniform float16 t, uniform float16 f); __declspec(safe, cost1) inline int32 select(bool cond, int32 t, int32 f); __declspec(safe, cost1) inline int32 select(uniform bool cond, int32 t, int32 f); __declspec(safe, cost1) inline uniform int32 select(uniform bool cond, uniform int32 t, uniform int32 f); __declspec(safe, cost1) inline int64 select(bool cond, int64 t, int64 f); __declspec(safe, cost1) inline int64 select(uniform bool cond, int64 t, int64 f); __declspec(safe, cost1) inline uniform int64 select(uniform bool cond, uniform int64 t, uniform int64 f); __declspec(safe, cost1) inline float select(bool cond, float t, float f); __declspec(safe, cost1) inline float select(uniform bool cond, float t, float f); __declspec(safe, cost1) inline uniform float select(uniform bool cond, uniform float t, uniform float f); __declspec(safe, cost1) inline double select(bool cond, double t, double f); __declspec(safe, cost1) inline double select(uniform bool cond, double t, double f); __declspec(safe, cost1) inline uniform double select(uniform bool cond, uniform double t, uniform double f); /////////////////////////////////////////////////////////////////////////// // Horizontal ops / reductions __declspec(safe) inline uniform int16 reduce_add(int8 x); __declspec(safe) inline uniform unsigned int16 reduce_add(unsigned int8 x); __declspec(safe) inline uniform int32 reduce_add(int16 x); __declspec(safe) inline uniform unsigned int32 reduce_add(unsigned int16 x); __declspec(safe) inline uniform float16 reduce_add(float16 x); __declspec(safe) inline uniform float16 reduce_min(float16 v); __declspec(safe) inline uniform float16 reduce_max(float16 v); __declspec(safe) inline uniform float reduce_add(float x); __declspec(safe) inline uniform float reduce_min(float v); __declspec(safe) inline uniform float reduce_max(float v); __declspec(safe) inline uniform int64 reduce_add(int32 x); __declspec(safe) inline uniform int reduce_min(int v); __declspec(safe) inline uniform int reduce_max(int v); __declspec(safe) inline uniform unsigned int64 reduce_add(unsigned int32 x); __declspec(safe) inline uniform unsigned int reduce_min(unsigned int v); __declspec(safe) inline uniform unsigned int reduce_max(unsigned int v); __declspec(safe) inline uniform double reduce_add(double x); __declspec(safe) inline uniform double reduce_min(double v); __declspec(safe) inline uniform double reduce_max(double v); __declspec(safe) inline uniform int64 reduce_add(int64 x); __declspec(safe) inline uniform int64 reduce_min(int64 v); __declspec(safe) inline uniform int64 reduce_max(int64 v); __declspec(safe) inline uniform unsigned int64 reduce_add(unsigned int64 x); __declspec(safe) inline uniform unsigned int64 reduce_min(unsigned int64 v); __declspec(safe) inline uniform unsigned int64 reduce_max(unsigned int64 v); #define REDUCE_EQUAL_DECL(TYPE, FUNCTYPE, MASKTYPE) \ __declspec(safe) inline uniform bool reduce_equal(TYPE v); \ __declspec(safe) inline uniform bool reduce_equal(TYPE v, uniform TYPE *uniform value); REDUCE_EQUAL_DECL(float16, half, UIntMaskType) REDUCE_EQUAL_DECL(int32, int32, UIntMaskType) REDUCE_EQUAL_DECL(unsigned int32, int32, UUIntMaskType) REDUCE_EQUAL_DECL(float, float, UIntMaskType) REDUCE_EQUAL_DECL(int64, int64, UIntMaskType) REDUCE_EQUAL_DECL(unsigned int64, int64, UUIntMaskType) REDUCE_EQUAL_DECL(double, double, UIntMaskType) #undef REDUCE_EQUAL_DECL inline float16 exclusive_scan_add(float16 v); inline int32 exclusive_scan_add(int32 v); inline unsigned int32 exclusive_scan_add(unsigned int32 v); inline float exclusive_scan_add(float v); inline int64 exclusive_scan_add(int64 v); inline unsigned int64 exclusive_scan_add(unsigned int64 v); inline double exclusive_scan_add(double v); inline int32 exclusive_scan_and(int32 v); inline unsigned int32 exclusive_scan_and(unsigned int32 v); inline int64 exclusive_scan_and(int64 v); inline unsigned int64 exclusive_scan_and(unsigned int64 v); inline int32 exclusive_scan_or(int32 v); inline unsigned int32 exclusive_scan_or(unsigned int32 v); inline int64 exclusive_scan_or(int64 v); inline unsigned int64 exclusive_scan_or(unsigned int64 v); /////////////////////////////////////////////////////////////////////////// // packed load, store /* unsigned int32 implementations. */ // unsigned int32 load. inline uniform int packed_load_active(uniform unsigned int a[], varying unsigned int *uniform vals); // unsigned int32 store. inline uniform int packed_store_active(uniform unsigned int a[], unsigned int vals); // unsigned int32 store2. inline uniform int packed_store_active2(uniform unsigned int a[], unsigned int vals); /* int32 implementations. */ // int32 load. inline uniform int packed_load_active(uniform int a[], varying int *uniform vals); // int32 store. inline uniform int packed_store_active(uniform int a[], int vals); // int32 store2. inline uniform int packed_store_active2(uniform int a[], int vals); // int32 store with lanes. inline uniform int packed_store_active(bool active, uniform int a[], int vals); /* unsigned int64 implementations. */ // unsigned int64 load. inline uniform int packed_load_active(uniform unsigned int64 a[], varying unsigned int64 *uniform vals); // unsigned int64 store. inline uniform int packed_store_active(uniform unsigned int64 a[], unsigned int64 vals); // unsigned int64 store2. inline uniform int packed_store_active2(uniform unsigned int64 a[], unsigned int64 vals); /* int64 implementations. */ // int64 load. inline uniform int packed_load_active(uniform int64 a[], varying int64 *uniform vals); // int64 store. inline uniform int packed_store_active(uniform int64 a[], int64 vals); // int64 store2. inline uniform int packed_store_active2(uniform int64 a[], int64 vals); // int64 store with lanes. inline uniform int packed_store_active(bool active, uniform int64 a[], int64 vals); /////////////////////////////////////////////////////////////////////////// // streaming store __declspec(safe, cost1) inline void streaming_store(uniform unsigned int8 a[], unsigned int8 vals); __declspec(safe, cost1) inline void streaming_store(uniform int8 a[], int8 vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int16 a[], unsigned int16 vals); __declspec(safe, cost1) inline void streaming_store(uniform int16 a[], int16 vals); __declspec(safe, cost1) inline void streaming_store(uniform float16 a[], float16 vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int a[], unsigned int vals); __declspec(safe, cost1) inline void streaming_store(uniform int a[], int vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int64 a[], unsigned int64 vals); __declspec(safe, cost1) inline void streaming_store(uniform int64 a[], int64 vals); __declspec(safe, cost1) inline void streaming_store(uniform float a[], float vals); __declspec(safe, cost1) inline void streaming_store(uniform double a[], double vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int8 a[], uniform unsigned int8 vals); __declspec(safe, cost1) inline void streaming_store(uniform int8 a[], uniform int8 vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int16 a[], uniform unsigned int16 vals); __declspec(safe, cost1) inline void streaming_store(uniform int16 a[], uniform int16 vals); __declspec(safe, cost1) inline void streaming_store(uniform float16 a[], uniform float16 vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int a[], uniform unsigned int vals); __declspec(safe, cost1) inline void streaming_store(uniform int a[], uniform int vals); __declspec(safe, cost1) inline void streaming_store(uniform unsigned int64 a[], uniform unsigned int64 vals); __declspec(safe, cost1) inline void streaming_store(uniform int64 a[], uniform int64 vals); __declspec(safe, cost1) inline void streaming_store(uniform float a[], uniform float vals); __declspec(safe, cost1) inline void streaming_store(uniform double a[], uniform double vals); /////////////////////////////////////////////////////////////////////////// // streaming load __declspec(safe, cost1) inline varying unsigned int8 streaming_load(uniform unsigned int8 a[]); __declspec(safe, cost1) inline varying int8 streaming_load(uniform int8 a[]); __declspec(safe, cost1) inline uniform unsigned int8 streaming_load_uniform(uniform unsigned int8 a[]); __declspec(safe, cost1) inline uniform int8 streaming_load_uniform(uniform int8 a[]); __declspec(safe, cost1) inline varying unsigned int16 streaming_load(uniform unsigned int16 a[]); __declspec(safe, cost1) inline varying int16 streaming_load(uniform int16 a[]); __declspec(safe, cost1) inline uniform unsigned int16 streaming_load_uniform(uniform unsigned int16 a[]); __declspec(safe, cost1) inline uniform int16 streaming_load_uniform(uniform int16 a[]); __declspec(safe, cost1) inline varying float16 streaming_load(uniform float16 a[]); __declspec(safe, cost1) inline uniform float16 streaming_load_uniform(uniform float16 a[]); __declspec(safe, cost1) inline varying unsigned int streaming_load(uniform unsigned int a[]); __declspec(safe, cost1) inline varying int streaming_load(uniform int a[]); __declspec(safe, cost1) inline uniform unsigned int streaming_load_uniform(uniform unsigned int a[]); __declspec(safe, cost1) inline uniform int streaming_load_uniform(uniform int a[]); __declspec(safe, cost1) inline varying unsigned int64 streaming_load(uniform unsigned int64 a[]); __declspec(safe, cost1) inline varying int64 streaming_load(uniform int64 a[]); __declspec(safe, cost1) inline uniform unsigned int64 streaming_load_uniform(uniform unsigned int64 a[]); __declspec(safe, cost1) inline uniform int64 streaming_load_uniform(uniform int64 a[]); __declspec(safe, cost1) inline varying float streaming_load(uniform float a[]); __declspec(safe, cost1) inline uniform float streaming_load_uniform(uniform float a[]); __declspec(safe, cost1) inline varying double streaming_load(uniform double a[]); __declspec(safe, cost1) inline uniform double streaming_load_uniform(uniform double a[]); /////////////////////////////////////////////////////////////////////////// // System information inline uniform int num_cores(); __declspec(safe) inline uniform int64 clock(); /////////////////////////////////////////////////////////////////////////// // Floating-Point Math __declspec(safe, cost1) inline uniform bool isnan(uniform float16 v); __declspec(safe, cost1) inline bool isnan(float16 v); __declspec(safe, cost1) inline uniform bool isnan(uniform float v); __declspec(safe, cost1) inline bool isnan(float v); __declspec(safe, cost1) inline uniform bool isnan(uniform double v); __declspec(safe, cost1) inline bool isnan(double v); __declspec(safe, cost1) inline int8 abs(int8 a); __declspec(safe, cost1) inline uniform int8 abs(uniform int8 a); __declspec(safe, cost1) inline int16 abs(int16 a); __declspec(safe, cost1) inline uniform int16 abs(uniform int16 a); __declspec(safe, cost1) inline int abs(int a); __declspec(safe, cost1) inline uniform int abs(uniform int a); __declspec(safe, cost1) inline int64 abs(int64 a); __declspec(safe, cost1) inline uniform int64 abs(uniform int64 a); __declspec(safe, cost1) inline float16 abs(float16 a); __declspec(safe, cost1) inline uniform float16 abs(uniform float16 a); __declspec(safe, cost1) inline float abs(float a); __declspec(safe, cost1) inline uniform float abs(uniform float a); __declspec(safe, cost1) inline double abs(double a); __declspec(safe, cost1) inline uniform double abs(uniform double a); __declspec(safe, cost1) inline unsigned int16 signbits(float16 x); __declspec(safe, cost1) inline uniform unsigned int16 signbits(uniform float16 x); __declspec(safe, cost1) inline unsigned int signbits(float x); __declspec(safe, cost1) inline uniform unsigned int signbits(uniform float x); __declspec(safe, cost1) inline unsigned int64 signbits(double x); __declspec(safe, cost1) inline uniform unsigned int64 signbits(uniform double x); __declspec(safe, cost2) inline float16 round(float16 x); __declspec(safe, cost2) inline uniform float16 round(uniform float16 x); __declspec(safe, cost2) inline float round(float x); __declspec(safe, cost2) inline uniform float round(uniform float x); __declspec(safe, cost2) inline double round(double x); __declspec(safe, cost2) inline uniform double round(uniform double x); __declspec(safe, cost2) inline float16 floor(float16 x); __declspec(safe, cost2) inline uniform float16 floor(uniform float16 x); __declspec(safe, cost2) inline float floor(float x); __declspec(safe, cost2) inline uniform float floor(uniform float x); __declspec(safe, cost2) inline double floor(double x); __declspec(safe, cost2) inline uniform double floor(uniform double x); __declspec(safe, cost2) inline float16 ceil(float16 x); __declspec(safe, cost2) inline uniform float16 ceil(uniform float16 x); __declspec(safe, cost2) inline float ceil(float x); __declspec(safe, cost2) inline uniform float ceil(uniform float x); __declspec(safe, cost2) inline double ceil(double x); __declspec(safe, cost2) inline uniform double ceil(uniform double x); /////////////////////////// __declspec(safe, cost2) inline float16 fmod(float16 x, float16 y); __declspec(safe, cost2) inline uniform float16 fmod(uniform float16 x, uniform float16 y); __declspec(safe, cost2) inline float fmod(float x, float y); __declspec(safe, cost2) inline uniform float fmod(uniform float x, uniform float y); __declspec(safe, cost2) inline double fmod(double x, double y); __declspec(safe, cost2) inline uniform double fmod(uniform double x, uniform double y); __declspec(safe, cost2) inline float16 trunc(float16 x); __declspec(safe, cost2) inline uniform float16 trunc(uniform float16 x); __declspec(safe, cost2) inline float trunc(float x); __declspec(safe, cost2) inline uniform float trunc(uniform float x); __declspec(safe, cost2) inline double trunc(double x); __declspec(safe, cost2) inline uniform double trunc(uniform double x); __declspec(safe) inline float rcp(float v); __declspec(safe) inline uniform float rcp(uniform float v); __declspec(safe) inline float rcp_fast(float v); __declspec(safe) inline uniform float rcp_fast(uniform float v); #define RCPD_DECL(QUAL) \ __declspec(safe) inline QUAL double __rcp_iterate_##QUAL##_double(QUAL double v, QUAL double iv); \ __declspec(safe) inline QUAL double __rcp_safe_##QUAL##_double(QUAL double x); RCPD_DECL(varying) __declspec(safe) inline double rcp(double v); RCPD_DECL(uniform) #undef RCPD_DECL __declspec(safe) inline uniform double rcp(uniform double v); __declspec(safe) inline double rcp_fast(double v); __declspec(safe) inline uniform double rcp_fast(uniform double v); __declspec(safe) inline float16 rcp(float16 v); __declspec(safe) inline uniform float16 rcp(uniform float16 v); /////////////////////////////////////////////////////////////////////////// // min/max // float16 __declspec(safe, cost1) inline float16 min(float16 a, float16 b); __declspec(safe, cost1) inline uniform float16 min(uniform float16 a, uniform float16 b); __declspec(safe, cost1) inline float16 max(float16 a, float16 b); __declspec(safe, cost1) inline uniform float16 max(uniform float16 a, uniform float16 b); // float __declspec(safe, cost1) inline float min(float a, float b); __declspec(safe, cost1) inline uniform float min(uniform float a, uniform float b); __declspec(safe, cost1) inline float max(float a, float b); __declspec(safe, cost1) inline uniform float max(uniform float a, uniform float b); // double __declspec(safe) inline double min(double a, double b); __declspec(safe) inline uniform double min(uniform double a, uniform double b); __declspec(safe) inline double max(double a, double b); __declspec(safe) inline uniform double max(uniform double a, uniform double b); // int8 __declspec(safe, cost1) inline uniform unsigned int8 min(uniform unsigned int8 a, uniform unsigned int8 b); __declspec(safe, cost1) inline uniform unsigned int8 max(uniform unsigned int8 a, uniform unsigned int8 b); __declspec(safe, cost1) inline uniform int8 min(uniform int8 a, uniform int8 b); __declspec(safe, cost1) inline uniform int8 max(uniform int8 a, uniform int8 b); __declspec(safe, cost1) inline unsigned int8 min(unsigned int8 a, unsigned int8 b); __declspec(safe, cost1) inline unsigned int8 max(unsigned int8 a, unsigned int8 b); __declspec(safe, cost1) inline int8 min(int8 a, int8 b); __declspec(safe, cost1) inline int8 max(int8 a, int8 b); // int16 __declspec(safe, cost1) inline uniform unsigned int16 min(uniform unsigned int16 a, uniform unsigned int16 b); __declspec(safe, cost1) inline uniform unsigned int16 max(uniform unsigned int16 a, uniform unsigned int16 b); __declspec(safe, cost1) inline uniform int16 min(uniform int16 a, uniform int16 b); __declspec(safe, cost1) inline uniform int16 max(uniform int16 a, uniform int16 b); __declspec(safe, cost1) inline unsigned int16 min(unsigned int16 a, unsigned int16 b); __declspec(safe, cost1) inline unsigned int16 max(unsigned int16 a, unsigned int16 b); __declspec(safe, cost1) inline int16 min(int16 a, int16 b); __declspec(safe, cost1) inline int16 max(int16 a, int16 b); // int32 __declspec(safe, cost1) inline unsigned int min(unsigned int a, unsigned int b); __declspec(safe, cost1) inline uniform unsigned int min(uniform unsigned int a, uniform unsigned int b); __declspec(safe, cost1) inline unsigned int max(unsigned int a, unsigned int b); __declspec(safe, cost1) inline uniform unsigned int max(uniform unsigned int a, uniform unsigned int b); __declspec(safe, cost1) inline int min(int a, int b); __declspec(safe, cost1) inline uniform int min(uniform int a, uniform int b); __declspec(safe, cost1) inline int max(int a, int b); __declspec(safe, cost1) inline uniform int max(uniform int a, uniform int b); // int64 __declspec(safe, cost1) inline unsigned int64 min(unsigned int64 a, unsigned int64 b); __declspec(safe, cost1) inline uniform unsigned int64 min(uniform unsigned int64 a, uniform unsigned int64 b); __declspec(safe, cost1) inline unsigned int64 max(unsigned int64 a, unsigned int64 b); __declspec(safe, cost1) inline uniform unsigned int64 max(uniform unsigned int64 a, uniform unsigned int64 b); __declspec(safe, cost1) inline int64 min(int64 a, int64 b); __declspec(safe, cost1) inline uniform int64 min(uniform int64 a, uniform int64 b); __declspec(safe, cost1) inline int64 max(int64 a, int64 b); __declspec(safe, cost1) inline uniform int64 max(uniform int64 a, uniform int64 b); /////////////////////////////////////////////////////////////////////////// // clamps // float16 __declspec(safe, cost2) inline float16 clamp(float16 v, float16 low, float16 high); __declspec(safe, cost2) inline uniform float16 clamp(uniform float16 v, uniform float16 low, uniform float16 high); // float __declspec(safe, cost2) inline float clamp(float v, float low, float high); __declspec(safe, cost2) inline uniform float clamp(uniform float v, uniform float low, uniform float high); // double __declspec(safe, cost2) inline double clamp(double v, double low, double high); __declspec(safe, cost2) inline uniform double clamp(uniform double v, uniform double low, uniform double high); // int8 __declspec(safe, cost2) inline unsigned int8 clamp(unsigned int8 v, unsigned int8 low, unsigned int8 high); __declspec(safe, cost2) inline uniform unsigned int8 clamp(uniform unsigned int8 v, uniform unsigned int8 low, uniform unsigned int8 high); __declspec(safe, cost2) inline int8 clamp(int8 v, int8 low, int8 high); __declspec(safe, cost2) inline uniform int8 clamp(uniform int8 v, uniform int8 low, uniform int8 high); // int16 __declspec(safe, cost2) inline unsigned int16 clamp(unsigned int16 v, unsigned int16 low, unsigned int16 high); __declspec(safe, cost2) inline uniform unsigned int16 clamp(uniform unsigned int16 v, uniform unsigned int16 low, uniform unsigned int16 high); __declspec(safe, cost2) inline int16 clamp(int16 v, int16 low, int16 high); __declspec(safe, cost2) inline uniform int16 clamp(uniform int16 v, uniform int16 low, uniform int16 high); // int32 __declspec(safe, cost2) inline unsigned int clamp(unsigned int v, unsigned int low, unsigned int high); __declspec(safe, cost2) inline uniform unsigned int clamp(uniform unsigned int v, uniform unsigned int low, uniform unsigned int high); __declspec(safe, cost2) inline int clamp(int v, int low, int high); __declspec(safe, cost2) inline uniform int clamp(uniform int v, uniform int low, uniform int high); // int64 __declspec(safe, cost2) inline unsigned int64 clamp(unsigned int64 v, unsigned int64 low, unsigned int64 high); __declspec(safe, cost2) inline uniform unsigned int64 clamp(uniform unsigned int64 v, uniform unsigned int64 low, uniform unsigned int64 high); __declspec(safe, cost2) inline int64 clamp(int64 v, int64 low, int64 high); __declspec(safe, cost2) inline uniform int64 clamp(uniform int64 v, uniform int64 low, uniform int64 high); /////////////////////////////////////////////////////////////////////////// // Global atomics and memory barriers inline void memory_barrier(); #define DEFINE_ATOMIC_OP_DECL(TA, TB, OPA, OPB, MASKTYPE, TC) \ inline TA atomic_##OPA##_global(uniform TA *uniform ptr, TA value); \ inline uniform TA atomic_##OPA##_global(uniform TA *uniform ptr, uniform TA value); \ inline TA atomic_##OPA##_global(uniform TA *varying ptr, TA value); #define DEFINE_ATOMIC_SWAP_DECL(TA, TB, MASKTYPE, TC) \ inline TA atomic_swap_global(uniform TA *uniform ptr, TA value); \ inline uniform TA atomic_swap_global(uniform TA *uniform ptr, uniform TA value); \ inline TA atomic_swap_global(uniform TA *varying ptr, TA value); #define DEFINE_ATOMIC_MINMAX_OP_DECL(TA, TB, OPA, OPB, MASKTYPE, TC) \ inline TA atomic_##OPA##_global(uniform TA *uniform ptr, TA value); \ inline uniform TA atomic_##OPA##_global(uniform TA *uniform ptr, uniform TA value); \ inline TA atomic_##OPA##_global(uniform TA *varying ptr, TA value); DEFINE_ATOMIC_OP_DECL(int32, int32, add, add, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int32, int32, subtract, sub, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(int32, int32, min, min, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(int32, int32, max, max, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int32, int32, and, and, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int32, int32, or, or, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int32, int32, xor, xor, IntMaskType, int64) DEFINE_ATOMIC_SWAP_DECL(int32, int32, IntMaskType, int64) // For everything but atomic min and max, we can use the same // implementations for unsigned as for signed. DEFINE_ATOMIC_OP_DECL(unsigned int32, int32, add, add, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int32, int32, subtract, sub, UIntMaskType, unsigned int64) DEFINE_ATOMIC_MINMAX_OP_DECL(unsigned int32, uint32, min, umin, UIntMaskType, unsigned int64) DEFINE_ATOMIC_MINMAX_OP_DECL(unsigned int32, uint32, max, umax, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int32, int32, and, and, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int32, int32, or, or, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int32, int32, xor, xor, UIntMaskType, unsigned int64) DEFINE_ATOMIC_SWAP_DECL(unsigned int32, int32, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(float, float, add, fadd, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(float, float, subtract, fsub, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(float, float, min, fmin, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(float, float, max, fmax, IntMaskType, int64) DEFINE_ATOMIC_SWAP_DECL(float, float, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int64, int64, add, add, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int64, int64, subtract, sub, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(int64, int64, min, min, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(int64, int64, max, max, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int64, int64, and, and, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int64, int64, or, or, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(int64, int64, xor, xor, IntMaskType, int64) DEFINE_ATOMIC_SWAP_DECL(int64, int64, IntMaskType, int64) // For everything but atomic min and max, we can use the same // implementations for unsigned as for signed. DEFINE_ATOMIC_OP_DECL(unsigned int64, int64, add, add, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int64, int64, subtract, sub, UIntMaskType, unsigned int64) DEFINE_ATOMIC_MINMAX_OP_DECL(unsigned int64, uint64, min, umin, UIntMaskType, unsigned int64) DEFINE_ATOMIC_MINMAX_OP_DECL(unsigned int64, uint64, max, umax, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int64, int64, and, and, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int64, int64, or, or, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(unsigned int64, int64, xor, xor, UIntMaskType, unsigned int64) DEFINE_ATOMIC_SWAP_DECL(unsigned int64, int64, UIntMaskType, unsigned int64) DEFINE_ATOMIC_OP_DECL(double, double, add, fadd, IntMaskType, int64) DEFINE_ATOMIC_OP_DECL(double, double, subtract, fsub, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(double, double, min, fmin, IntMaskType, int64) DEFINE_ATOMIC_MINMAX_OP_DECL(double, double, max, fmax, IntMaskType, int64) DEFINE_ATOMIC_SWAP_DECL(double, double, IntMaskType, int64) #undef DEFINE_ATOMIC_OP_DECL #undef DEFINE_ATOMIC_MINMAX_OP_DECL #undef DEFINE_ATOMIC_SWAP_DECL #define ATOMIC_DECL_CMPXCHG_DECL(TA, TB, MASKTYPE, TC) \ inline uniform TA atomic_compare_exchange_global(uniform TA *uniform ptr, uniform TA oldval, uniform TA newval); \ inline TA atomic_compare_exchange_global(uniform TA *uniform ptr, TA oldval, TA newval); \ inline TA atomic_compare_exchange_global(uniform TA *varying ptr, TA oldval, TA newval); ATOMIC_DECL_CMPXCHG_DECL(int32, int32, IntMaskType, int64) ATOMIC_DECL_CMPXCHG_DECL(unsigned int32, int32, UIntMaskType, unsigned int64) ATOMIC_DECL_CMPXCHG_DECL(float, float, IntMaskType, int64) ATOMIC_DECL_CMPXCHG_DECL(int64, int64, IntMaskType, int64) ATOMIC_DECL_CMPXCHG_DECL(unsigned int64, int64, UIntMaskType, unsigned int64) ATOMIC_DECL_CMPXCHG_DECL(double, double, IntMaskType, int64) #undef ATOMIC_DECL_CMPXCHG_DECL // void * variants of swap and compare exchange inline void *atomic_swap_global(void **uniform ptr, void *value); inline void *uniform atomic_swap_global(void **uniform ptr, void *uniform value); inline void *atomic_swap_global(void **ptr, void *value); inline void *atomic_compare_exchange_global(void **uniform ptr, void *oldval, void *newval); inline void *uniform atomic_compare_exchange_global(void **uniform ptr, void *uniform oldval, void *uniform newval); inline void *atomic_compare_exchange_global(void **ptr, void *oldval, void *newval); /////////////////////////////////////////////////////////////////////////// // local atomics #define LOCAL_ATOMIC_DECL(TYPE, NAME, OPFUNC) \ inline uniform TYPE atomic_##NAME##_local(uniform TYPE *uniform ptr, uniform TYPE value); \ inline TYPE atomic_##NAME##_local(uniform TYPE *uniform ptr, TYPE value); \ inline TYPE atomic_##NAME##_local(uniform TYPE *p, TYPE value); inline uniform int32 __add(uniform int32 a, uniform int32 b); inline uniform int32 __sub(uniform int32 a, uniform int32 b); inline uniform int32 __and(uniform int32 a, uniform int32 b); inline uniform int32 __or(uniform int32 a, uniform int32 b); inline uniform int32 __xor(uniform int32 a, uniform int32 b); inline uniform int32 __swap(uniform int32 a, uniform int32 b); inline uniform unsigned int32 __add(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform unsigned int32 __sub(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform unsigned int32 __and(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform unsigned int32 __or(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform unsigned int32 __xor(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform unsigned int32 __swap(uniform unsigned int32 a, uniform unsigned int32 b); inline uniform float __add(uniform float a, uniform float b); inline uniform float __sub(uniform float a, uniform float b); inline uniform float __swap(uniform float a, uniform float b); inline uniform int64 __add(uniform int64 a, uniform int64 b); inline uniform int64 __sub(uniform int64 a, uniform int64 b); inline uniform int64 __and(uniform int64 a, uniform int64 b); inline uniform int64 __or(uniform int64 a, uniform int64 b); inline uniform int64 __xor(uniform int64 a, uniform int64 b); inline uniform int64 __swap(uniform int64 a, uniform int64 b); inline uniform unsigned int64 __add(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform unsigned int64 __sub(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform unsigned int64 __and(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform unsigned int64 __or(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform unsigned int64 __xor(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform unsigned int64 __swap(uniform unsigned int64 a, uniform unsigned int64 b); inline uniform double __add(uniform double a, uniform double b); inline uniform double __sub(uniform double a, uniform double b); inline uniform double __swap(uniform double a, uniform double b); LOCAL_ATOMIC_DECL(int32, add, __add) LOCAL_ATOMIC_DECL(int32, subtract, __sub) LOCAL_ATOMIC_DECL(int32, and, __and) LOCAL_ATOMIC_DECL(int32, or, __or) LOCAL_ATOMIC_DECL(int32, xor, __xor) LOCAL_ATOMIC_DECL(int32, min, min) LOCAL_ATOMIC_DECL(int32, max, max) LOCAL_ATOMIC_DECL(int32, swap, __swap) LOCAL_ATOMIC_DECL(unsigned int32, add, __add) LOCAL_ATOMIC_DECL(unsigned int32, subtract, __sub) LOCAL_ATOMIC_DECL(unsigned int32, and, __and) LOCAL_ATOMIC_DECL(unsigned int32, or, __or) LOCAL_ATOMIC_DECL(unsigned int32, xor, __xor) LOCAL_ATOMIC_DECL(unsigned int32, min, min) LOCAL_ATOMIC_DECL(unsigned int32, max, max) LOCAL_ATOMIC_DECL(unsigned int32, swap, __swap) LOCAL_ATOMIC_DECL(float, add, __add) LOCAL_ATOMIC_DECL(float, subtract, __sub) LOCAL_ATOMIC_DECL(float, min, min) LOCAL_ATOMIC_DECL(float, max, max) LOCAL_ATOMIC_DECL(float, swap, __swap) LOCAL_ATOMIC_DECL(int64, add, __add) LOCAL_ATOMIC_DECL(int64, subtract, __sub) LOCAL_ATOMIC_DECL(int64, and, __and) LOCAL_ATOMIC_DECL(int64, or, __or) LOCAL_ATOMIC_DECL(int64, xor, __xor) LOCAL_ATOMIC_DECL(int64, min, min) LOCAL_ATOMIC_DECL(int64, max, max) LOCAL_ATOMIC_DECL(int64, swap, __swap) LOCAL_ATOMIC_DECL(unsigned int64, add, __add) LOCAL_ATOMIC_DECL(unsigned int64, subtract, __sub) LOCAL_ATOMIC_DECL(unsigned int64, and, __and) LOCAL_ATOMIC_DECL(unsigned int64, or, __or) LOCAL_ATOMIC_DECL(unsigned int64, xor, __xor) LOCAL_ATOMIC_DECL(unsigned int64, min, min) LOCAL_ATOMIC_DECL(unsigned int64, max, max) LOCAL_ATOMIC_DECL(unsigned int64, swap, __swap) LOCAL_ATOMIC_DECL(double, add, __add) LOCAL_ATOMIC_DECL(double, subtract, __sub) LOCAL_ATOMIC_DECL(double, min, min) LOCAL_ATOMIC_DECL(double, max, max) LOCAL_ATOMIC_DECL(double, swap, __swap) // compare exchange #define LOCAL_CMPXCHG_DECL(TYPE) \ inline uniform TYPE atomic_compare_exchange_local(uniform TYPE *uniform ptr, uniform TYPE cmp, \ uniform TYPE update); \ inline TYPE atomic_compare_exchange_local(uniform TYPE *uniform ptr, TYPE cmp, TYPE update); \ inline TYPE atomic_compare_exchange_local(uniform TYPE *varying p, TYPE cmp, TYPE update); LOCAL_CMPXCHG_DECL(int32) LOCAL_CMPXCHG_DECL(unsigned int32) LOCAL_CMPXCHG_DECL(float) LOCAL_CMPXCHG_DECL(int64) LOCAL_CMPXCHG_DECL(unsigned int64) LOCAL_CMPXCHG_DECL(double) #undef LOCAL_ATOMIC_DECL #undef LOCAL_CMPXCHG_DECL // void * variants of swap and compare exchange inline void *atomic_swap_local(void **uniform ptr, void *value); inline void *uniform atomic_swap_local(void **uniform ptr, void *uniform value); inline void *atomic_swap_local(void **ptr, void *value); inline void *atomic_compare_exchange_local(void **uniform ptr, void *oldval, void *newval); inline void *uniform atomic_compare_exchange_local(void **uniform ptr, void *uniform oldval, void *uniform newval); inline void *atomic_compare_exchange_local(void **ptr, void *oldval, void *newval); // Transcendentals (float precision) __declspec(safe) inline float sqrt(float v); __declspec(safe) inline uniform float sqrt(uniform float v); __declspec(safe) inline float rsqrt(float v); __declspec(safe) inline uniform float rsqrt(uniform float v); __declspec(safe) inline float rsqrt_fast(float v); __declspec(safe) inline uniform float rsqrt_fast(uniform float v); __declspec(safe) inline float ldexp(float x, int n); __declspec(safe) inline uniform float ldexp(uniform float x, uniform int n); __declspec(safe) inline float frexp(float x, varying int *uniform pw2); __declspec(safe) inline uniform float frexp(uniform float x, uniform int *uniform pw2); // Most of the transcendental implementations in ispc code here come from // Solomon Boulos's "syrah": https://github.com/boulos/syrah/ __declspec(safe) inline float sin(float x_full); __declspec(safe) inline uniform float sin(uniform float x_full); __declspec(safe) inline float asin(float x0); __declspec(safe) inline uniform float asin(uniform float x0); __declspec(safe) inline float cos(float x_full); __declspec(safe) inline uniform float cos(uniform float x_full); __declspec(safe) inline float acos(float v); __declspec(safe) inline uniform float acos(uniform float v); __declspec(safe) inline void sincos(float x_full, varying float *uniform sin_result, varying float *uniform cos_result); __declspec(safe) inline void sincos(uniform float x_full, uniform float *uniform sin_result, uniform float *uniform cos_result); __declspec(safe) inline float tan(float x_full); __declspec(safe) inline uniform float tan(uniform float x_full); __declspec(safe) inline float atan(float x_full); __declspec(safe) inline uniform float atan(uniform float x_full); __declspec(safe) inline float atan2(float y, float x); __declspec(safe) inline uniform float atan2(uniform float y, uniform float x); __declspec(safe) inline float exp(float x_full); __declspec(safe) inline uniform float exp(uniform float x_full); // Range reduction for logarithms takes log(x) -> log(2^n * y) -> n // * log(2) + log(y) where y is the reduced range (usually in [1/2, // 1)). __declspec(safe) inline void __range_reduce_log(float input, varying float *uniform reduced, varying int *uniform exponent); __declspec(safe) inline void __range_reduce_log(uniform float input, uniform float *uniform reduced, uniform int *uniform exponent); __declspec(safe) inline float log(float x_full); __declspec(safe) inline uniform float log(uniform float x_full); __declspec(safe) inline float pow(float a, float b); __declspec(safe) inline uniform float pow(uniform float a, uniform float b); /////////////////////////////////////////////////////////////////////////// // Transcendentals (16-bit float precision) __declspec(safe) inline float16 sqrt(float16 v); __declspec(safe) inline uniform float16 sqrt(uniform float16 v); __declspec(safe) inline float16 rsqrt(float16 v); __declspec(safe) inline uniform float16 rsqrt(uniform float16 v); __declspec(safe) inline float16 ldexp(float16 x, int n); __declspec(safe) inline uniform float16 ldexp(uniform float16 x, uniform int n); __declspec(safe) inline float16 frexp(float16 x, varying int *uniform pw2); __declspec(safe) inline uniform float16 frexp(uniform float16 x, uniform int *uniform pw2); // If no native trigonometry support, convert to float, get asin and convert to half back __declspec(safe) inline float16 sin(float16 x_full); __declspec(safe) inline uniform float16 sin(uniform float16 x_full); __declspec(safe) inline float16 asin(float16 x_full); __declspec(safe) inline uniform float16 asin(uniform float16 x_full); __declspec(safe) inline float16 cos(float16 x_full); __declspec(safe) inline uniform float16 cos(uniform float16 x_full); __declspec(safe) inline float16 tan(float16 x_full); __declspec(safe) inline uniform float16 tan(uniform float16 x_full); __declspec(safe) inline float16 acos(float16 x_full); __declspec(safe) inline uniform float16 acos(uniform float16 x_full); __declspec(safe) inline void sincos(float16 x_full, varying float16 *uniform sin_result, varying float16 *uniform cos_result); __declspec(safe) inline void sincos(uniform float16 x_full, uniform float16 *uniform sin_result, uniform float16 *uniform cos_result); __declspec(safe) inline float16 atan(float16 x_full); __declspec(safe) inline uniform float16 atan(uniform float16 x_full); __declspec(safe) inline float16 atan2(float16 y, float16 x); __declspec(safe) inline uniform float16 atan2(uniform float16 y, uniform float16 x); __declspec(safe) inline float16 exp(float16 x_full); __declspec(safe) inline uniform float16 exp(uniform float16 x_full); __declspec(safe) inline float16 log(float16 x_full); __declspec(safe) inline uniform float16 log(uniform float16 x_full); __declspec(safe) inline float16 pow(float16 a, float16 b); __declspec(safe) inline uniform float16 pow(uniform float16 a, uniform float16 b); /////////////////////////////////////////////////////////////////////////// // Transcendentals (double precision) __declspec(safe) inline double sqrt(double v); __declspec(safe) inline uniform double sqrt(uniform double v); #define RSQRTD_DECL(QUAL) \ __declspec(safe) inline QUAL double __rsqrt_iterate_##QUAL##_double(QUAL double x, QUAL double y); \ __declspec(safe) inline QUAL double __rsqrt_safe_##QUAL##_double(QUAL double x); RSQRTD_DECL(varying) __declspec(safe) inline double rsqrt(double v); RSQRTD_DECL(uniform) #undef RSQRTD_DECL __declspec(safe) inline uniform double rsqrt(uniform double v); __declspec(safe) inline double rsqrt_fast(double v); __declspec(safe) inline uniform double rsqrt_fast(uniform double v); __declspec(safe) inline double ldexp(double x, int n); __declspec(safe) inline uniform double ldexp(uniform double x, uniform int n); __declspec(safe) inline double frexp(double x, varying int *uniform pw2); __declspec(safe) inline uniform double frexp(uniform double x, uniform int *uniform pw2); __declspec(safe) inline double sin(double x); __declspec(safe) inline uniform double sin(uniform double x); __declspec(safe) inline uniform double asin(uniform double x); __declspec(safe) inline double asin(const double x); __declspec(safe) inline double cos(const double x); __declspec(safe) inline uniform double cos(uniform double x); __declspec(safe) inline double acos(const double v); __declspec(safe) inline uniform double acos(const uniform double v); __declspec(safe) inline void sincos(double x, varying double *uniform sin_result, varying double *uniform cos_result); __declspec(safe) inline void sincos(uniform double x, uniform double *uniform sin_result, uniform double *uniform cos_result); __declspec(safe) inline double tan(double x); __declspec(safe) inline uniform double tan(uniform double x); __declspec(safe) inline double atan(double x); __declspec(safe) inline uniform double atan(uniform double x); __declspec(safe) inline double atan2(double y, double x); __declspec(safe) inline uniform double atan2(uniform double y, uniform double x); __declspec(safe) inline double exp(double x); __declspec(safe) inline uniform double exp(uniform double x); __declspec(safe) inline double log(double x); __declspec(safe) inline uniform double log(uniform double x); __declspec(safe) inline double pow(double a, double b); __declspec(safe) inline uniform double pow(uniform double a, uniform double b); /////////////////////////////////////////////////////////////////////////// // half-precision floats __declspec(safe) inline uniform float half_to_float(uniform unsigned int16 h); __declspec(safe) inline float half_to_float(unsigned int16 h); __declspec(safe) inline uniform int16 float_to_half(uniform float f); __declspec(safe) inline int16 float_to_half(float f); __declspec(safe) inline uniform float half_to_float_fast(uniform unsigned int16 h); __declspec(safe) inline float half_to_float_fast(unsigned int16 h); __declspec(safe) inline uniform int16 float_to_half_fast(uniform float f); __declspec(safe) inline int16 float_to_half_fast(float f); /////////////////////////////////////////////////////////////////////////// // float -> srgb8 // https://gist.github.com/2246678, from Fabian "rygorous" Giesen. // // The basic ideas are still the same, only this time, we squeeze // everything into the table, even the linear part of the range; since we // are approximating the function as piecewise linear anyway, this is // fairly easy. // // In the exact version of the conversion, any value that produces an // output float less than 0.5 will be rounded to an integer of // zero. Inverting the linear part of the transform, we get: // // log2(0.5 / (255 * 12.92)) =~ -12.686 // // which in turn means that any value smaller than about 2^(-12.687) will // return 0. What this means is that we can adapt the clamping code to // just clamp to [2^(-13), 1-eps] and we're covered. This means our table // needs to cover a range of 13 different exponents from -13 to -1. // // The table lookup, storage and interpolation works exactly the same way // as in the code above. // // Max error for the whole function (integer-rounded result minus "exact" // value, as computed in floats using the official formula): 0.544403 at // 0x3e9f8000 __declspec(safe) inline int float_to_srgb8(float inval); __declspec(safe) inline uniform int float_to_srgb8(uniform float inval); /////////////////////////////////////////////////////////////////////////// // RNG stuff inline unsigned int random(varying RNGState *uniform state); inline uniform unsigned int random(uniform RNGState *uniform state); inline float frandom(varying RNGState *uniform state); inline uniform float frandom(uniform RNGState *uniform state); inline void seed_rng(varying RNGState *uniform state, unsigned int seed); inline void seed_rng(uniform RNGState *uniform state, uniform unsigned int seed); inline void fastmath(); /////////////////////////////////////////////////////////////////////////// // saturation arithmetic inline uniform int8 saturating_add(uniform int8 a, uniform int8 b); inline varying int8 saturating_add(varying int8 a, varying int8 b); inline uniform int16 saturating_add(uniform int16 a, uniform int16 b); inline varying int16 saturating_add(varying int16 a, varying int16 b); inline uniform int32 saturating_add(uniform int32 a, uniform int32 b); inline varying int32 saturating_add(varying int32 a, varying int32 b); inline uniform int64 saturating_add(uniform int64 a, uniform int64 b); inline varying int64 saturating_add(varying int64 a, varying int64 b); inline uniform unsigned int8 saturating_add(uniform unsigned int8 a, uniform unsigned int8 b); inline varying unsigned int8 saturating_add(varying unsigned int8 a, varying unsigned int8 b); inline uniform unsigned int16 saturating_add(uniform unsigned int16 a, uniform unsigned int16 b); inline varying unsigned int16 saturating_add(varying unsigned int16 a, varying unsigned int16 b); inline uniform unsigned int32 saturating_add(uniform unsigned int32 a, uniform unsigned int32 b); inline varying unsigned int32 saturating_add(varying unsigned int32 a, varying unsigned int32 b); inline uniform unsigned int64 saturating_add(uniform unsigned int64 a, uniform unsigned int64 b); inline varying unsigned int64 saturating_add(varying unsigned int64 a, varying unsigned int64 b); inline uniform int8 saturating_sub(uniform int8 a, uniform int8 b); inline varying int8 saturating_sub(varying int8 a, varying int8 b); inline uniform int16 saturating_sub(uniform int16 a, uniform int16 b); inline varying int16 saturating_sub(varying int16 a, varying int16 b); inline uniform int32 saturating_sub(uniform int32 a, uniform int32 b); inline varying int32 saturating_sub(varying int32 a, varying int32 b); inline uniform int64 saturating_sub(uniform int64 a, uniform int64 b); inline varying int64 saturating_sub(varying int64 a, varying int64 b); inline uniform unsigned int8 saturating_sub(uniform unsigned int8 a, uniform unsigned int8 b); inline varying unsigned int8 saturating_sub(varying unsigned int8 a, varying unsigned int8 b); inline uniform unsigned int16 saturating_sub(uniform unsigned int16 a, uniform unsigned int16 b); inline varying unsigned int16 saturating_sub(varying unsigned int16 a, varying unsigned int16 b); inline uniform unsigned int32 saturating_sub(uniform unsigned int32 a, uniform unsigned int32 b); inline varying unsigned int32 saturating_sub(varying unsigned int32 a, varying unsigned int32 b); inline uniform unsigned int64 saturating_sub(uniform unsigned int64 a, uniform unsigned int64 b); inline varying unsigned int64 saturating_sub(varying unsigned int64 a, varying unsigned int64 b); inline uniform int8 saturating_div(uniform int8 a, uniform int8 b); inline varying int8 saturating_div(varying int8 a, varying int8 b); inline uniform int16 saturating_div(uniform int16 a, uniform int16 b); inline varying int16 saturating_div(varying int16 a, varying int16 b); inline uniform int32 saturating_div(uniform int32 a, uniform int32 b); inline varying int32 saturating_div(varying int32 a, varying int32 b); inline uniform int64 saturating_div(uniform int64 a, uniform int64 b); inline varying int64 saturating_div(varying int64 a, varying int64 b); inline uniform unsigned int8 saturating_div(uniform unsigned int8 a, uniform unsigned int8 b); inline varying unsigned int8 saturating_div(varying unsigned int8 a, varying unsigned int8 b); inline uniform unsigned int16 saturating_div(uniform unsigned int16 a, uniform unsigned int16 b); inline varying unsigned int16 saturating_div(varying unsigned int16 a, varying unsigned int16 b); inline uniform unsigned int32 saturating_div(uniform unsigned int32 a, uniform unsigned int32 b); inline varying unsigned int32 saturating_div(varying unsigned int32 a, varying unsigned int32 b); inline uniform unsigned int64 saturating_div(uniform unsigned int64 a, uniform unsigned int64 b); inline varying unsigned int64 saturating_div(varying unsigned int64 a, varying unsigned int64 b); inline uniform int8 saturating_mul(uniform int8 a, uniform int8 b); inline varying int8 saturating_mul(varying int8 a, varying int8 b); inline uniform int16 saturating_mul(uniform int16 a, uniform int16 b); inline varying int16 saturating_mul(varying int16 a, varying int16 b); inline uniform int32 saturating_mul(uniform int32 a, uniform int32 b); inline varying int32 saturating_mul(varying int32 a, varying int32 b); inline uniform unsigned int8 saturating_mul(uniform unsigned int8 a, uniform unsigned int8 b); inline varying unsigned int8 saturating_mul(varying unsigned int8 a, varying unsigned int8 b); inline uniform unsigned int16 saturating_mul(uniform unsigned int16 a, uniform unsigned int16 b); inline varying unsigned int16 saturating_mul(varying unsigned int16 a, varying unsigned int16 b); inline uniform unsigned int32 saturating_mul(uniform unsigned int32 a, uniform unsigned int32 b); inline varying unsigned int32 saturating_mul(varying unsigned int32 a, varying unsigned int32 b); inline uniform int64 saturating_mul(uniform int64 a, uniform int64 b); inline varying int64 saturating_mul(varying int64 a, varying int64 b); inline uniform unsigned int64 saturating_mul(uniform unsigned int64 a, uniform unsigned int64 b); inline varying unsigned int64 saturating_mul(varying unsigned int64 a, varying unsigned int64 b); /////////////////////////////////////////////////////////////////////////// // rdrand inline uniform bool rdrand(float *uniform ptr); inline bool rdrand(varying float *uniform ptr); inline bool rdrand(float *ptr); inline uniform bool rdrand(int16 *uniform ptr); inline bool rdrand(varying int16 *uniform ptr); inline bool rdrand(int16 *ptr); inline uniform bool rdrand(int32 *uniform ptr); inline bool rdrand(varying int32 *uniform ptr); inline bool rdrand(int32 *ptr); inline uniform bool rdrand(int64 *uniform ptr); inline bool rdrand(varying int64 *uniform ptr); inline bool rdrand(int64 *ptr); /////////////////////////////////////////////////////////////////////////// // Saturating int8/int16 ops __declspec(safe) inline unmasked unsigned int8 avg_up(unsigned int8 a, unsigned int8 b); __declspec(safe) inline unmasked int8 avg_up(int8 a, int8 b); __declspec(safe) inline unmasked unsigned int16 avg_up(unsigned int16 a, unsigned int16 b); __declspec(safe) inline unmasked int16 avg_up(int16 a, int16 b); __declspec(safe) inline unmasked unsigned int8 avg_down(unsigned int8 a, unsigned int8 b); __declspec(safe) inline unmasked int8 avg_down(int8 a, int8 b); __declspec(safe) inline unmasked unsigned int16 avg_down(unsigned int16 a, unsigned int16 b); __declspec(safe) inline unmasked int16 avg_down(int16 a, int16 b); /////////////////////////////////////////////////////////////////////////// // Assume uniform/varying ops __declspec(safe) inline void assume(uniform bool test); /////////////////////////////////////////////////////////////////////////// // Dot product and accumulate // Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc, and return the result. __declspec(safe) inline varying int32 dot4add_u8i8packed(varying uint32 a, varying uint32 b, varying int32 acc); // Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc using signed saturation, and return the result. __declspec(safe) inline varying int32 dot4add_u8i8packed_sat(varying uint32 a, varying uint32 b, varying int32 acc); // Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding unsigned // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc, and return the result. __declspec(safe) inline varying uint32 dot4add_u8u8packed(varying uint32 a, varying uint32 b, varying uint32 acc); // Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding unsigned // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc using unsigned saturation, and return the result. __declspec(safe) inline varying uint32 dot4add_u8u8packed_sat(varying uint32 a, varying uint32 b, varying uint32 acc); // Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding signed // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc, and return the result. __declspec(safe) inline varying int32 dot4add_i8i8packed(varying uint32 a, varying uint32 b, varying int32 acc); // Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding signed // 8-bit integers in b, producing 4 intermediate signed 16-bit results. // Sum these 4 results with the corresponding 32-bit integer in acc using signed saturation, and return the result. __declspec(safe) inline varying int32 dot4add_i8i8packed_sat(varying uint32 a, varying uint32 b, varying int32 acc); // Multiply groups of 2 adjacent pairs of signed 16-bit integers in a // with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. // Sum these 2 results with the corresponding 32-bit integer in src, and return the result. __declspec(safe) inline varying int32 dot2add_i16packed(varying uint32 a, varying uint32 b, varying int32 acc); // Multiply groups of 2 adjacent pairs of signed 16-bit integers in a // with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. // Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and return the result. __declspec(safe) inline varying int32 dot2add_i16packed_sat(varying uint32 a, varying uint32 b, varying int32 acc);