opencv

1
/* adler32_avx512.c -- compute the Adler-32 checksum of a data stream
2
 * Copyright (C) 1995-2011 Mark Adler
3
 * Authors:
4
 *   Adam Stylinski <kungfujesus06@gmail.com>
5
 *   Brian Bockelman <bockelman@gmail.com>
6
 * For conditions of distribution and use, see copyright notice in zlib.h
7
 */
8

9
#ifdef X86_AVX512
10

11
#include "../../zbuild.h"
12
#include "../../adler32_p.h"
13
#include "../../adler32_fold.h"
14
#include "../../cpu_features.h"
15
#include <immintrin.h>
16
#include "x86_intrins.h"
17
#include "adler32_avx512_p.h"
18

19
static inline uint32_t adler32_fold_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) {
20
    if (src == NULL) return 1L;
21
    if (len == 0) return adler;
22

23
    uint32_t adler0, adler1;
24
    adler1 = (adler >> 16) & 0xffff;
25
    adler0 = adler & 0xffff;
26

27
rem_peel:
28
    if (len < 64) {
29
        /* This handles the remaining copies, just call normal adler checksum after this */
30
        if (COPY) {
31
            __mmask64 storemask = (0xFFFFFFFFFFFFFFFFUL >> (64 - len));
32
            __m512i copy_vec = _mm512_maskz_loadu_epi8(storemask, src);
33
            _mm512_mask_storeu_epi8(dst, storemask, copy_vec);
34
        }
35

36
#ifdef X86_AVX2
37
        return adler32_avx2(adler, src, len);
38
#elif defined(X86_SSSE3)
39
        return adler32_ssse3(adler, src, len);
40
#else
41
        return adler32_len_16(adler0, src, len, adler1);
42
#endif
43
    }
44

45
    __m512i vbuf, vs1_0, vs3;
46

47
    const __m512i dot2v = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
48
                                          20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
49
                                          38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
50
                                          56, 57, 58, 59, 60, 61, 62, 63, 64);
51
    const __m512i dot3v = _mm512_set1_epi16(1);
52
    const __m512i zero = _mm512_setzero_si512();
53
    size_t k;
54

55
    while (len >= 64) {
56
        __m512i vs1 = _mm512_zextsi128_si512(_mm_cvtsi32_si128(adler0));
57
        __m512i vs2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128(adler1));
58
        vs1_0 = vs1;
59
        vs3 = _mm512_setzero_si512();
60

61
        k = MIN(len, NMAX);
62
        k -= k % 64;
63
        len -= k;
64

65
        while (k >= 64) {
66
            /*
67
               vs1 = adler + sum(c[i])
68
               vs2 = sum2 + 64 vs1 + sum( (64-i+1) c[i] )
69
            */
70
            vbuf = _mm512_loadu_si512(src);
71

72
            if (COPY) {
73
                _mm512_storeu_si512(dst, vbuf);
74
                dst += 64;
75
            }
76

77
            src += 64;
78
            k -= 64;
79

80
            __m512i vs1_sad = _mm512_sad_epu8(vbuf, zero);
81
            __m512i v_short_sum2 = _mm512_maddubs_epi16(vbuf, dot2v);
82
            vs1 = _mm512_add_epi32(vs1_sad, vs1);
83
            vs3 = _mm512_add_epi32(vs3, vs1_0);
84
            __m512i vsum2 = _mm512_madd_epi16(v_short_sum2, dot3v);
85
            vs2 = _mm512_add_epi32(vsum2, vs2);
86
            vs1_0 = vs1;
87
        }
88

89
        vs3 = _mm512_slli_epi32(vs3, 6);
90
        vs2 = _mm512_add_epi32(vs2, vs3);
91

92
        adler0 = partial_hsum(vs1) % BASE;
93
        adler1 = _mm512_reduce_add_epu32(vs2) % BASE;
94
    }
95

96
    adler = adler0 | (adler1 << 16);
97

98
    /* Process tail (len < 64). */
99
    if (len) {
100
        goto rem_peel;
101
    }
102

103
    return adler;
104
}
105

106
Z_INTERNAL uint32_t adler32_fold_copy_avx512(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
107
    return adler32_fold_copy_impl(adler, dst, src, len, 1);
108
}
109

110
Z_INTERNAL uint32_t adler32_avx512(uint32_t adler, const uint8_t *src, size_t len) {
111
    return adler32_fold_copy_impl(adler, NULL, src, len, 0);
112
}
113

114
#endif
115

116