ncnn

1
// Tencent is pleased to support the open source community by making ncnn available.
2
//
3
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
4
//
5
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6
// in compliance with the License. You may obtain a copy of the License at
7
//
8
// https://opensource.org/licenses/BSD-3-Clause
9
//
10
// Unless required by applicable law or agreed to in writing, software distributed
11
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
13
// specific language governing permissions and limitations under the License.
14

15
#include "pipelinecache.h"
16

17
#include "gpu.h"
18

19
namespace ncnn {
20

21
#if NCNN_VULKAN
22
// https://en.wikipedia.org/wiki/MurmurHash
23
static uint32_t murmur3_32(const uint32_t* data, int size)
24
{
25
    uint32_t h = 0;
26

27
    for (int i = 0; i < size; i++)
28
    {
29
        uint32_t k = *data++;
30

31
        k *= 0xcc9e2d51;
32
        k = (k << 15) | (k >> (32 - 15));
33
        k *= 0x1b873593;
34

35
        h ^= k;
36
        h = (h << 13) | (h >> (32 - 13));
37
        h = (h * 5) + 0xe6546b64;
38
    }
39

40
    h ^= size * 4;
41

42
    h ^= h >> 16;
43
    h *= 0x85ebca6b;
44
    h ^= h >> 13;
45
    h *= 0xc2b2ae35;
46
    h ^= h >> 16;
47

48
    return h;
49
}
50

51
// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function#FNV-1a_hash
52
static uint32_t fnv1a_32(const uint8_t* data, int size)
53
{
54
    uint32_t h = 0x811c9dc5;
55

56
    for (int i = 0; i < size; i++)
57
    {
58
        h ^= (uint32_t)*data++;
59
        h *= 0x01000193;
60
    }
61

62
    return h;
63
}
64

65
class PipelineCachePrivate
66
{
67
public:
68
    // digest -> artifact
69
    struct pipeline_cache_digest
70
    {
71
        pipeline_cache_digest(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
72
                              uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z);
73
        pipeline_cache_digest(int shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
74
                              uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z);
75

76
        bool operator==(const pipeline_cache_digest& rhs) const
77
        {
78
            return d0 == rhs.d0 && d1 == rhs.d1;
79
        }
80

81
        bool operator!=(const pipeline_cache_digest& rhs) const
82
        {
83
            return d0 != rhs.d0 || d1 != rhs.d1;
84
        }
85

86
        union
87
        {
88
            struct
89
            {
90
                union
91
                {
92
                    uint32_t spv_data_murmur3;
93
                    int shader_type_index;
94
                };
95
                unsigned char opt_local_size_bits[4];
96
            };
97

98
            uint64_t d0;
99
        };
100

101
        union
102
        {
103
            struct
104
            {
105
                uint32_t specializations_murmur3;
106
                uint32_t specializations_fnv1a;
107
            };
108

109
            uint64_t d1;
110
        };
111
    };
112

113
    struct pipeline_cache_artifact
114
    {
115
        VkShaderModule shader_module;
116
        VkDescriptorSetLayout descriptorset_layout;
117
        VkPipelineLayout pipeline_layout;
118
        VkPipeline pipeline;
119
        VkDescriptorUpdateTemplateKHR descriptor_update_template;
120
        ShaderInfo shader_info; // TODO use pointer ?
121
    };
122

123
    mutable std::vector<pipeline_cache_digest> cache_digests;
124
    mutable std::vector<pipeline_cache_artifact> cache_artifacts;
125
    mutable Mutex cache_lock;
126
};
127

128
PipelineCachePrivate::pipeline_cache_digest::pipeline_cache_digest(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
129
        uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z)
130
{
131
    spv_data_murmur3 = murmur3_32(spv_data, spv_data_size / 4);
132

133
    // encode opt
134
    opt_local_size_bits[0] = 0;
135

136
    // encode local_size
137
    opt_local_size_bits[1] = local_size_x;
138
    opt_local_size_bits[2] = local_size_y;
139
    opt_local_size_bits[3] = local_size_z;
140

141
    // encode specializations
142
    const int specialization_count = specializations.size();
143
    specializations_murmur3 = murmur3_32((const uint32_t*)specializations.data(), specialization_count);
144
    specializations_fnv1a = fnv1a_32((const uint8_t*)specializations.data(), specialization_count * sizeof(vk_specialization_type));
145
}
146

147
PipelineCachePrivate::pipeline_cache_digest::pipeline_cache_digest(int _shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
148
        uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z)
149
{
150
    shader_type_index = _shader_type_index;
151

152
    // encode opt
153
    opt_local_size_bits[0] = opt.use_image_storage << 7
154
                             | opt.use_fp16_packed << 6
155
                             | opt.use_fp16_storage << 5
156
                             | opt.use_fp16_arithmetic << 4
157
                             | opt.use_int8_storage << 3
158
                             | opt.use_int8_arithmetic << 2;
159

160
    // encode local_size
161
    opt_local_size_bits[1] = local_size_x;
162
    opt_local_size_bits[2] = local_size_y;
163
    opt_local_size_bits[3] = local_size_z;
164

165
    // encode specializations
166
    const int specialization_count = specializations.size();
167
    specializations_murmur3 = murmur3_32((const uint32_t*)specializations.data(), specialization_count);
168
    specializations_fnv1a = fnv1a_32((const uint8_t*)specializations.data(), specialization_count * sizeof(vk_specialization_type));
169
}
170

171
PipelineCache::PipelineCache(const VulkanDevice* _vkdev)
172
    : vkdev(_vkdev), d(new PipelineCachePrivate)
173
{
174
}
175

176
PipelineCache::~PipelineCache()
177
{
178
    clear();
179

180
    delete d;
181
}
182

183
PipelineCache::PipelineCache(const PipelineCache&)
184
    : d(0)
185
{
186
}
187

188
PipelineCache& PipelineCache::operator=(const PipelineCache&)
189
{
190
    return *this;
191
}
192

193
void PipelineCache::clear()
194
{
195
    MutexLockGuard lock(d->cache_lock);
196

197
    for (size_t i = 0; i < d->cache_artifacts.size(); i++)
198
    {
199
        const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
200

201
        if (vkdev->info.support_VK_KHR_descriptor_update_template())
202
        {
203
            if (cc.descriptor_update_template)
204
            {
205
                vkdev->vkDestroyDescriptorUpdateTemplateKHR(vkdev->vkdevice(), cc.descriptor_update_template, 0);
206
            }
207
        }
208

209
        if (cc.pipeline)
210
        {
211
            vkDestroyPipeline(vkdev->vkdevice(), cc.pipeline, 0);
212
        }
213

214
        if (cc.pipeline_layout)
215
        {
216
            vkDestroyPipelineLayout(vkdev->vkdevice(), cc.pipeline_layout, 0);
217
        }
218

219
        if (cc.descriptorset_layout)
220
        {
221
            vkDestroyDescriptorSetLayout(vkdev->vkdevice(), cc.descriptorset_layout, 0);
222
        }
223

224
        if (cc.shader_module)
225
        {
226
            vkDestroyShaderModule(vkdev->vkdevice(), cc.shader_module, 0);
227
        }
228
    }
229

230
    d->cache_digests.clear();
231
    d->cache_artifacts.clear();
232
}
233

234
int PipelineCache::get_pipeline(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
235
                                uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
236
                                VkShaderModule* _shader_module,
237
                                VkDescriptorSetLayout* descriptorset_layout,
238
                                VkPipelineLayout* pipeline_layout,
239
                                VkPipeline* pipeline,
240
                                VkDescriptorUpdateTemplateKHR* descriptor_update_template,
241
                                ShaderInfo& shader_info) const
242
{
243
    MutexLockGuard lock(d->cache_lock);
244

245
    PipelineCachePrivate::pipeline_cache_digest key(spv_data, spv_data_size, specializations, local_size_x, local_size_y, local_size_z);
246

247
    if (!vkdev->info.bug_corrupted_online_pipeline_cache())
248
    {
249
        // find cache
250
        for (size_t i = 0; i < d->cache_digests.size(); i++)
251
        {
252
            if (d->cache_digests[i] != key)
253
                continue;
254

255
            // hit cache
256
            const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
257

258
            *_shader_module = cc.shader_module;
259
            *descriptorset_layout = cc.descriptorset_layout;
260
            *pipeline_layout = cc.pipeline_layout;
261
            *pipeline = cc.pipeline;
262
            *descriptor_update_template = cc.descriptor_update_template;
263
            shader_info = cc.shader_info;
264

265
            // NCNN_LOGE("get_pipeline hit %d", last_digest_index);
266

267
            return 0;
268
        }
269
    }
270

271
    int ret = 0;
272

273
    ret = resolve_shader_info(spv_data, spv_data_size, shader_info);
274
    if (ret != 0)
275
    {
276
        NCNN_LOGE("resolve_shader_info failed %d", ret);
277
        return -1;
278
    }
279

280
    VkShaderModule shader_module = vkdev->compile_shader_module(spv_data, spv_data_size, local_size_x, local_size_y, local_size_z);
281
    if (!shader_module)
282
    {
283
        NCNN_LOGE("create_shader_module failed");
284
        return -1;
285
    }
286

287
    ret = new_pipeline(shader_module, shader_info, specializations, descriptorset_layout, pipeline_layout, pipeline, descriptor_update_template);
288
    if (ret != 0)
289
    {
290
        NCNN_LOGE("new_pipeline failed");
291
        vkDestroyShaderModule(vkdev->vkdevice(), shader_module, 0);
292
        return -1;
293
    }
294

295
    *_shader_module = shader_module;
296

297
    // save to cache
298
    {
299
        PipelineCachePrivate::pipeline_cache_artifact cc;
300

301
        cc.shader_module = *_shader_module;
302
        cc.descriptorset_layout = *descriptorset_layout;
303
        cc.pipeline_layout = *pipeline_layout;
304
        cc.pipeline = *pipeline;
305
        cc.descriptor_update_template = *descriptor_update_template;
306
        cc.shader_info = shader_info;
307

308
        d->cache_digests.push_back(key);
309
        d->cache_artifacts.push_back(cc);
310
    }
311

312
    // NCNN_LOGE("new_pipeline %d", last_digest_index);
313

314
    return 0;
315
}
316

317
int PipelineCache::get_pipeline(int shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
318
                                uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
319
                                VkShaderModule* _shader_module,
320
                                VkDescriptorSetLayout* descriptorset_layout,
321
                                VkPipelineLayout* pipeline_layout,
322
                                VkPipeline* pipeline,
323
                                VkDescriptorUpdateTemplateKHR* descriptor_update_template,
324
                                ShaderInfo& shader_info) const
325
{
326
    MutexLockGuard lock(d->cache_lock);
327

328
    PipelineCachePrivate::pipeline_cache_digest key(shader_type_index, opt, specializations, local_size_x, local_size_y, local_size_z);
329

330
    if (!vkdev->info.bug_corrupted_online_pipeline_cache())
331
    {
332
        // find cache
333
        for (size_t i = 0; i < d->cache_digests.size(); i++)
334
        {
335
            if (d->cache_digests[i] != key)
336
                continue;
337

338
            // hit cache
339
            const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
340

341
            *_shader_module = cc.shader_module;
342
            *descriptorset_layout = cc.descriptorset_layout;
343
            *pipeline_layout = cc.pipeline_layout;
344
            *pipeline = cc.pipeline;
345
            *descriptor_update_template = cc.descriptor_update_template;
346
            shader_info = cc.shader_info;
347

348
            // NCNN_LOGE("get_pipeline hit %d", last_digest_index);
349

350
            return 0;
351
        }
352
    }
353

354
    int ret = 0;
355

356
    // create new pipeline
357
    VkShaderModule shader_module = 0;
358
    ret = create_shader_module(shader_type_index, opt, local_size_x, local_size_y, local_size_z, &shader_module, shader_info);
359
    if (ret != 0)
360
    {
361
        NCNN_LOGE("create_shader_module failed");
362
        return -1;
363
    }
364

365
    ret = new_pipeline(shader_module, shader_info, specializations, descriptorset_layout, pipeline_layout, pipeline, descriptor_update_template);
366
    if (ret != 0)
367
    {
368
        NCNN_LOGE("new_pipeline failed");
369
        vkDestroyShaderModule(vkdev->vkdevice(), shader_module, 0);
370
        return -1;
371
    }
372

373
    *_shader_module = shader_module;
374

375
    // save to cache
376
    {
377
        PipelineCachePrivate::pipeline_cache_artifact cc;
378

379
        cc.shader_module = *_shader_module;
380
        cc.descriptorset_layout = *descriptorset_layout;
381
        cc.pipeline_layout = *pipeline_layout;
382
        cc.pipeline = *pipeline;
383
        cc.descriptor_update_template = *descriptor_update_template;
384
        cc.shader_info = shader_info;
385

386
        d->cache_digests.push_back(key);
387
        d->cache_artifacts.push_back(cc);
388
    }
389

390
    // NCNN_LOGE("new_pipeline %d", last_digest_index);
391

392
    return 0;
393
}
394

395
int PipelineCache::create_shader_module(int shader_type_index, const Option& opt, uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
396
                                        VkShaderModule* _shader_module, ShaderInfo& si) const
397
{
398
    std::vector<uint32_t> spirv;
399
    int retc = compile_spirv_module(shader_type_index, opt, spirv);
400
    if (retc != 0)
401
    {
402
        NCNN_LOGE("compile_spirv_module failed %d", retc);
403
        return -1;
404
    }
405

406
    const uint32_t* spv_data = spirv.data();
407
    size_t spv_data_size = spirv.size() * 4;
408

409
    int ret = resolve_shader_info(spv_data, spv_data_size, si);
410
    if (ret != 0)
411
    {
412
        NCNN_LOGE("resolve_shader_info failed %d", ret);
413
        return -1;
414
    }
415

416
    VkShaderModule shader_module = vkdev->compile_shader_module(spv_data, spv_data_size, local_size_x, local_size_y, local_size_z);
417

418
    if (!shader_module)
419
    {
420
        NCNN_LOGE("create_shader_module failed");
421
        return -1;
422
    }
423

424
    *_shader_module = shader_module;
425

426
    return 0;
427
}
428

429
int PipelineCache::new_pipeline(VkShaderModule shader_module, const ShaderInfo& shader_info, const std::vector<vk_specialization_type>& specializations,
430
                                VkDescriptorSetLayout* _descriptorset_layout,
431
                                VkPipelineLayout* _pipeline_layout,
432
                                VkPipeline* _pipeline,
433
                                VkDescriptorUpdateTemplateKHR* _descriptor_update_template) const
434
{
435
    int ret = 0;
436

437
    VkDescriptorSetLayout descriptorset_layout = 0;
438
    VkPipelineLayout pipeline_layout = 0;
439
    VkPipeline pipeline = 0;
440
    VkDescriptorUpdateTemplateKHR descriptor_update_template = 0;
441

442
    // create new pipeline
443
    if ((int)specializations.size() != shader_info.specialization_count)
444
    {
445
        NCNN_LOGE("pipeline specialization count mismatch, expect %d but got %d", shader_info.specialization_count, (int)specializations.size());
446
        goto ERROR_PipelineCache;
447
    }
448

449
    ret = vkdev->create_descriptorset_layout(shader_info.binding_count, shader_info.binding_types, &descriptorset_layout);
450
    if (ret != 0)
451
        goto ERROR_PipelineCache;
452

453
    ret = vkdev->create_pipeline_layout(shader_info.push_constant_count, descriptorset_layout, &pipeline_layout);
454
    if (ret != 0)
455
        goto ERROR_PipelineCache;
456

457
    ret = vkdev->create_pipeline(shader_module, pipeline_layout, specializations, &pipeline);
458
    if (ret != 0)
459
        goto ERROR_PipelineCache;
460

461
    if (vkdev->info.support_VK_KHR_descriptor_update_template())
462
    {
463
        ret = vkdev->create_descriptor_update_template(shader_info.binding_count, shader_info.binding_types, descriptorset_layout, pipeline_layout, &descriptor_update_template);
464
        if (ret != 0)
465
            goto ERROR_PipelineCache;
466
    }
467

468
    *_descriptorset_layout = descriptorset_layout;
469
    *_pipeline_layout = pipeline_layout;
470
    *_pipeline = pipeline;
471
    *_descriptor_update_template = descriptor_update_template;
472

473
    return 0;
474

475
ERROR_PipelineCache:
476

477
    if (vkdev->info.support_VK_KHR_descriptor_update_template())
478
    {
479
        if (descriptor_update_template)
480
        {
481
            vkdev->vkDestroyDescriptorUpdateTemplateKHR(vkdev->vkdevice(), descriptor_update_template, 0);
482
        }
483
    }
484

485
    if (pipeline)
486
    {
487
        vkDestroyPipeline(vkdev->vkdevice(), pipeline, 0);
488
    }
489

490
    if (pipeline_layout)
491
    {
492
        vkDestroyPipelineLayout(vkdev->vkdevice(), pipeline_layout, 0);
493
    }
494

495
    if (descriptorset_layout)
496
    {
497
        vkDestroyDescriptorSetLayout(vkdev->vkdevice(), descriptorset_layout, 0);
498
    }
499

500
    return -1;
501
}
502

503
#endif // NCNN_VULKAN
504

505
} // namespace ncnn
506