ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#version 450
16

17
#if NCNN_fp16_storage
18
#extension GL_EXT_shader_16bit_storage: require
19
#endif
20
#if NCNN_fp16_arithmetic
21
#extension GL_EXT_shader_explicit_arithmetic_types_float16: require
22
#endif
23

24
layout (constant_id = 0) const int attn_mask = 0;
25
layout (constant_id = 1) const int M = 0;
26
layout (constant_id = 2) const int N = 0;
27
layout (constant_id = 3) const int K = 0;
28
layout (constant_id = 4) const int B = 0;
29
layout (constant_id = 5) const int attn_mask_dims = 0;
30

31
#if NCNN_image_shader
32
layout (binding = 0) uniform unfp sampler3D q_blob_3d;
33
layout (binding = 1) uniform unfp sampler3D k_blob_3d;
34
layout (binding = 2, imfmtc1) writeonly uniform unfp image3D qkcross_blob_3d;
35
layout (binding = 3) uniform unfp sampler3D attn_mask_blob_3d;
36
#else
37
layout (binding = 0) readonly buffer q_blob { sfpvec4 q_blob_data[]; };
38
layout (binding = 1) readonly buffer k_blob { sfpvec4 k_blob_data[]; };
39
layout (binding = 2) writeonly buffer qkcross_blob { sfp qkcross_blob_data[]; };
40
layout (binding = 3) readonly buffer attn_mask_blob { sfp attn_mask_blob_data[]; };
41
#endif
42

43
layout (push_constant) uniform parameter
44
{
45
    int M;
46
    int N;
47
    int K;
48
    int B;
49
    int attn_mask_dims;
50
} p;
51

52
void main()
53
{
54
    int gx = int(gl_GlobalInvocationID.x);
55
    int gy = int(gl_GlobalInvocationID.y);
56
    int gz = int(gl_GlobalInvocationID.z);
57

58
    if (gx >= psc(N) || gy >= psc(M) || gz >= psc(B))
59
        return;
60

61
    afp sum = afp(0.f);
62

63
    if (attn_mask == 1)
64
    {
65
        if (psc(attn_mask_dims) == 3)
66
        {
67
#if NCNN_image_shader
68
            sum = image3d_ld1(attn_mask_blob_3d, ivec3(gx, gy, gz));
69
#else
70
            const int mi = gz * psc(M) * psc(N) + gy * psc(N) + gx;
71
            sum = buffer_ld1(attn_mask_blob_data, mi);
72
#endif
73
        }
74
        else
75
        {
76
#if NCNN_image_shader
77
            sum = image3d_ld1(attn_mask_blob_3d, ivec3(gx, gy, 0));
78
#else
79
            const int mi = gy * psc(N) + gx;
80
            sum = buffer_ld1(attn_mask_blob_data, mi);
81
#endif
82
        }
83
    }
84

85
    for (int k = 0; k < psc(K); k++)
86
    {
87
#if NCNN_image_shader
88
        afpvec4 q0 = image3d_ld4(q_blob_3d, ivec3(gy, gz * psc(K) + k, 0));
89

90
        afpvec4 k0 = image3d_ld4(k_blob_3d, ivec3(gx, gz * psc(K) + k, 0));
91
#else
92
        const int ai = gz * psc(M) * psc(K) + k * psc(M) + gy;
93
        afpvec4 q0 = buffer_ld4(q_blob_data, ai);
94

95
        const int bi = gz * psc(N) * psc(K) + k * psc(N) + gx;
96
        afpvec4 k0 = buffer_ld4(k_blob_data, bi);
97
#endif
98

99
        sum += dot(q0, k0);
100
    }
101

102
#if NCNN_image_shader
103
    image3d_st1(qkcross_blob_3d, ivec3(gx, gz * psc(M) + gy, 0), sum);
104
#else
105
    const int gi = gz * psc(M) * psc(N) + gy * psc(N) + gx;
106
    buffer_st1(qkcross_blob_data, gi, sum);
107
#endif
108
}
109