ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#version 450
16

17
#if NCNN_fp16_storage
18
#extension GL_EXT_shader_16bit_storage: require
19
#endif
20
#if NCNN_fp16_arithmetic
21
#extension GL_EXT_shader_explicit_arithmetic_types_float16: require
22
#endif
23

24
layout (constant_id = 0) const int attn_mask = 0;
25
layout (constant_id = 1) const int M = 0;
26
layout (constant_id = 2) const int N = 0;
27
layout (constant_id = 3) const int K = 0;
28
layout (constant_id = 4) const int B = 0;
29
layout (constant_id = 5) const int attn_mask_dims = 0;
30

31
#if NCNN_image_shader
32
layout (binding = 0) uniform unfp sampler3D q_blob_3d;
33
layout (binding = 1) uniform unfp sampler3D k_blob_3d;
34
layout (binding = 2, imfmtc4) writeonly uniform unfp image3D qkcross_blob_3d;
35
layout (binding = 3) uniform unfp sampler3D attn_mask_blob_3d;
36
#else
37
layout (binding = 0) readonly buffer q_blob { sfp q_blob_data[]; };
38
layout (binding = 1) readonly buffer k_blob { sfp k_blob_data[]; };
39
layout (binding = 2) writeonly buffer qkcross_blob { sfpvec4 qkcross_blob_data[]; };
40
layout (binding = 3) readonly buffer attn_mask_blob { sfpvec4 attn_mask_blob_data[]; };
41
#endif
42

43
layout (push_constant) uniform parameter
44
{
45
    int M;
46
    int N;
47
    int K;
48
    int B;
49
    int attn_mask_dims;
50
} p;
51

52
void main()
53
{
54
    int gx = int(gl_GlobalInvocationID.x);
55
    int gy = int(gl_GlobalInvocationID.y);
56
    int gz = int(gl_GlobalInvocationID.z);
57

58
    if (gx >= psc(N) || gy >= psc(M) || gz >= psc(B))
59
        return;
60

61
    afpvec4 sum = afpvec4(0.f);
62

63
    if (attn_mask == 1)
64
    {
65
        if (psc(attn_mask_dims) == 3)
66
        {
67
#if NCNN_image_shader
68
            sum = image3d_ld4(attn_mask_blob_3d, ivec3(gx, gy, gz));
69
#else
70
            const int mi = gz * psc(M) * psc(N) + gy * psc(N) + gx;
71
            sum = buffer_ld4(attn_mask_blob_data, mi);
72
#endif
73
        }
74
        else
75
        {
76
#if NCNN_image_shader
77
            sum = image3d_ld4(attn_mask_blob_3d, ivec3(gx, gy, 0));
78
#else
79
            const int mi = gy * psc(N) + gx;
80
            sum = buffer_ld4(attn_mask_blob_data, mi);
81
#endif
82
        }
83
    }
84

85
    for (int k = 0; k < psc(K); k++)
86
    {
87
#if NCNN_image_shader
88
        afp q0 = image3d_ld1(q_blob_3d, ivec3(gy * 4, gz * psc(K) + k, 0));
89
        afp q1 = image3d_ld1(q_blob_3d, ivec3(gy * 4 + 1, gz * psc(K) + k, 0));
90
        afp q2 = image3d_ld1(q_blob_3d, ivec3(gy * 4 + 2, gz * psc(K) + k, 0));
91
        afp q3 = image3d_ld1(q_blob_3d, ivec3(gy * 4 + 3, gz * psc(K) + k, 0));
92

93
        afp k0 = image3d_ld1(k_blob_3d, ivec3(gx, gz * psc(K) + k, 0));
94
#else
95
        const int ai = (gz * psc(M) * psc(K) + k * psc(M) + gy) * 4;
96
        afp q0 = buffer_ld1(q_blob_data, ai);
97
        afp q1 = buffer_ld1(q_blob_data, ai + 1);
98
        afp q2 = buffer_ld1(q_blob_data, ai + 2);
99
        afp q3 = buffer_ld1(q_blob_data, ai + 3);
100

101
        const int bi = gz * psc(N) * psc(K) + k * psc(N) + gx;
102
        afp k0 = buffer_ld1(k_blob_data, bi);
103
#endif
104

105
        sum.r += q0 * k0;
106
        sum.g += q1 * k0;
107
        sum.b += q2 * k0;
108
        sum.a += q3 * k0;
109
    }
110

111
#if NCNN_image_shader
112
    image3d_st4(qkcross_blob_3d, ivec3(gx, gz * psc(M) + gy, 0), sum);
113
#else
114
    const int gi = gz * psc(M) * psc(N) + gy * psc(N) + gx;
115
    buffer_st4(qkcross_blob_data, gi, sum);
116
#endif
117
}
118