/
dnezhkin
/
llama
GigaCode
beta
ОбзорЦентр заботыВойти

llama

Форк
0
Ветки: 328Коммиты: 3789Теги: 2441
llama
/tests
/
test-sampling.cpp 
320 строк · 13.0 Кб
1
#include "ggml.h"
2
#include "llama.h"
3
#include "llama-sampling.h"
4


5
#ifdef NDEBUG
6
#undef NDEBUG
7
#endif
8


9
#include <algorithm>
10
#include <cmath>
11
#include <string>
12
#include <vector>
13


14
static void dump(const llama_token_data_array * cur_p) {
15
    for (size_t i = 0; i < cur_p->size; i++) {
16
        printf("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
17
    }
18
}
19


20
#define DUMP(__cur_p) do { printf("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); printf("-\n"); } while(0)
21


22
#define APPLY(__cnstr, __cur_p) do { \
23
    auto * cnstr = (__cnstr); \
24
    llama_sampler_apply(cnstr, (__cur_p)); \
25
    llama_sampler_free(cnstr); \
26
} while(0)
27


28
static void test_top_k(const std::vector<float> & probs, const std::vector<float> & expected_probs, int k) {
29
    const size_t n_vocab = probs.size();
30


31
    std::vector<llama_token_data> cur;
32
    cur.reserve(n_vocab);
33
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
34
        const float logit = logf(probs[token_id]);
35
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
36
    }
37


38
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
39
    APPLY(llama_sampler_init_softmax(), &cur_p);
40
    DUMP(&cur_p);
41
    APPLY(llama_sampler_init_top_k(k), &cur_p);
42
    DUMP(&cur_p);
43


44
    GGML_ASSERT(cur_p.size == expected_probs.size());
45
    for (size_t i = 0; i < cur_p.size; i++) {
46
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-5);
47
    }
48
}
49


50
static void test_top_p(const std::vector<float> & probs, const std::vector<float> & expected_probs, float p) {
51
    const size_t n_vocab = probs.size();
52


53
    std::vector<llama_token_data> cur;
54
    cur.reserve(n_vocab);
55
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
56
        const float logit = logf(probs[token_id]);
57
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
58
    }
59


60
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
61
    APPLY(llama_sampler_init_softmax(), &cur_p);
62
    DUMP(&cur_p);
63
    APPLY(llama_sampler_init_top_p(p, 1), &cur_p);
64
    DUMP(&cur_p);
65


66
    GGML_ASSERT(cur_p.size == expected_probs.size());
67
    for (size_t i = 0; i < cur_p.size; i++) {
68
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-3);
69
    }
70
}
71


72
static void test_tfs(const std::vector<float> & probs, const std::vector<float> & expected_probs, float z) {
73
    const size_t n_vocab = probs.size();
74


75
    std::vector<llama_token_data> cur;
76
    cur.reserve(n_vocab);
77
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
78
        const float logit = logf(probs[token_id]);
79
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
80
    }
81


82
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
83
    DUMP(&cur_p);
84
    APPLY(llama_sampler_init_tail_free(z, 1), &cur_p);
85
    DUMP(&cur_p);
86


87
    GGML_ASSERT(cur_p.size == expected_probs.size());
88
    for (size_t i = 0; i < cur_p.size; i++) {
89
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-3);
90
    }
91
}
92


93
static void test_min_p(const std::vector<float> & probs, const std::vector<float> & expected_probs, float p) {
94
    const size_t n_vocab = probs.size();
95


96
    std::vector<llama_token_data> cur;
97
    cur.reserve(n_vocab);
98
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
99
        const float logit = logf(probs[token_id]);
100
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
101
    }
102


103
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
104
    DUMP(&cur_p);
105
    APPLY(llama_sampler_init_min_p(p, 1), &cur_p);
106
    DUMP(&cur_p);
107
    APPLY(llama_sampler_init_softmax(), &cur_p);
108


109
    GGML_ASSERT(cur_p.size == expected_probs.size());
110
    for (size_t i = 0; i < cur_p.size; i++) {
111
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-3);
112
    }
113
}
114


115
static void test_typical(const std::vector<float> & probs, const std::vector<float> & expected_probs, float p) {
116
    const size_t n_vocab = probs.size();
117


118
    std::vector<llama_token_data> cur;
119
    cur.reserve(n_vocab);
120
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
121
        const float logit = logf(probs[token_id]);
122
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
123
    }
124


125
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
126
    DUMP(&cur_p);
127
    APPLY(llama_sampler_init_typical(p, 1), &cur_p);
128
    DUMP(&cur_p);
129


130
    GGML_ASSERT(cur_p.size == expected_probs.size());
131
    for (size_t i = 0; i < cur_p.size; i++) {
132
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-3);
133
    }
134
}
135


136
static void test_penalties(
137
    const std::vector<float> & probs, const std::vector<llama_token> & last_tokens,
138
    const std::vector<float> & expected_probs, float repeat_penalty, float alpha_frequency, float alpha_presence
139
) {
140
    GGML_ASSERT(probs.size() == expected_probs.size());
141


142
    const size_t n_vocab = probs.size();
143


144
    std::vector<llama_token_data> cur;
145
    cur.reserve(n_vocab);
146
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
147
        const float logit = logf(probs[token_id]);
148
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
149
    }
150


151
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
152


153
    auto * sampler = llama_sampler_init_penalties(n_vocab, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence, false, false);
154


155
    for (size_t i = 0; i < last_tokens.size(); i++) {
156
        llama_sampler_accept(sampler, last_tokens[i]);
157
    }
158


159
    APPLY(llama_sampler_init_softmax(), &cur_p);
160
    DUMP(&cur_p);
161
    APPLY(sampler, &cur_p);
162
    APPLY(llama_sampler_init_softmax(), &cur_p);
163
    DUMP(&cur_p);
164


165
    GGML_ASSERT(cur_p.size == expected_probs.size());
166
    for (size_t i = 0; i < cur_p.size; i++) {
167
        GGML_ASSERT(fabs(cur_p.data[i].p - expected_probs[i]) < 1e-3);
168
    }
169
}
170


171
static void test_sampler_queue(const size_t n_vocab, const std::string & samplers_sequence, const int top_k, const float top_p, const float min_p
172
) {
173
    std::vector<llama_token_data> cur;
174
    cur.reserve(n_vocab);
175
    for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
176
        const float logit = logf(token_id);
177
        cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
178
    }
179


180
    llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
181


182
          llama_token min_token_id = 0;
183
    const llama_token max_token_id = n_vocab-1;
184


185
    for (auto s : samplers_sequence) {
186
        switch (s){
187
            case 'k': APPLY(llama_sampler_init_top_k(top_k), &cur_p); break;
188
            case 'f': GGML_ABORT("tail_free test not implemented");
189
            case 'y': GGML_ABORT("typical test not implemented");
190
            case 'p': APPLY(llama_sampler_init_top_p(top_p, 1), &cur_p); break;
191
            case 'm': APPLY(llama_sampler_init_min_p(min_p, 1), &cur_p); break;
192
            case 't': GGML_ABORT("temperature test not implemented");
193
            default : GGML_ABORT("Unknown sampler");
194
        }
195


196
        APPLY(llama_sampler_init_softmax(), &cur_p); // make sure tokens are sorted for tests
197


198
        const int size = cur_p.size;
199


200
        if (s == 'k') {
201
            const int expected_size = std::min(size, top_k);
202
            min_token_id = std::max(min_token_id, (llama_token)(n_vocab - top_k));
203


204
            GGML_ASSERT(size == expected_size);
205
            GGML_ASSERT(cur_p.data[0].id == max_token_id);
206
            GGML_ASSERT(cur_p.data[expected_size-1].id == min_token_id);
207
        } else if (s == 'p') {
208
            const int softmax_divisor = n_vocab * (n_vocab-1) / 2 - min_token_id * (min_token_id-1) / 2;
209
            const int softmax_numerator_target = ceilf(top_p * softmax_divisor);
210


211
                min_token_id  = n_vocab;
212
            int expected_size = 0;
213
            int cumsum        = 0;
214
            do { // do-while because always at least one token is sampled
215
                min_token_id--;
216
                expected_size++;
217


218
                cumsum += min_token_id;
219
            } while (cumsum < softmax_numerator_target);
220


221
            // token 0 has p == 0, need special consideration for cumsum because top_p immediately returns
222
            if (min_token_id == 1) {
223
                min_token_id--;
224
                expected_size += 1;
225
            }
226


227
            GGML_ASSERT(size == expected_size);
228
            GGML_ASSERT(cur_p.data[0].id == max_token_id);
229
            GGML_ASSERT(cur_p.data[expected_size-1].id == min_token_id);
230
        } else if (s == 'm') {
231
            int expected_size = ceilf((1.0f-min_p) * n_vocab);
232
            expected_size = std::max(expected_size, 1);
233
            expected_size = std::min(expected_size, size);
234


235
            min_token_id = floorf(min_p * n_vocab);
236
            min_token_id = std::max(min_token_id, 1);
237
            min_token_id = std::max(min_token_id, (llama_token)(n_vocab - size));
238
            min_token_id = std::min(min_token_id, (llama_token)(n_vocab - 1));
239


240
            GGML_ASSERT(size == expected_size);
241
            GGML_ASSERT(cur_p.data[0].id == max_token_id);
242
            GGML_ASSERT(cur_p.data[expected_size-1].id == min_token_id);
243
        } else {
244
            GGML_ABORT("fatal error");
245
        }
246
    }
247


248
    printf("Sampler queue %3s OK with n_vocab=%05zu top_k=%05d top_p=%f min_p=%f\n",
249
           samplers_sequence.c_str(), n_vocab, top_k, top_p, min_p);
250
}
251


252
int main(void) {
253
    ggml_time_init();
254


255
    test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f}, 1);
256
    test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f}, 3);
257
    test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 4);
258
    test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 0);
259


260
    test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f}, 0);
261
    test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f}, 0.7f);
262
    test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f}, 0.8f);
263
    test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 1);
264


265
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/1.0f, 0.3f/1.0f, 0.2f/1.0f, 0.1f/1.0f}, 0.00f);
266
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/1.0f, 0.3f/1.0f, 0.2f/1.0f, 0.1f/1.0f}, 0.24f);
267
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.9f, 0.3f/0.9f, 0.2f/0.9f},            0.26f);
268
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.9f, 0.3f/0.9f, 0.2f/0.9f},            0.49f);
269
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.7f, 0.3f/0.7f},                       0.51f);
270
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.7f, 0.3f/0.7f},                       0.74f);
271
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f},                                  0.76f);
272
    test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f},                                  1.00f);
273


274
    test_tfs({0.1f, 0.15f, 0.2f, 0.25f, 0.3f}, {0.3f}, 0.25f);
275
    test_tfs({0.1f, 0.15f, 0.2f, 0.25f, 0.3f}, {0.3f, 0.25f}, 0.75f);
276
    test_tfs({0.1f, 0.15f, 0.2f, 0.25f, 0.3f}, {0.3f, 0.25f}, 0.99f);
277


278
    test_typical({0.97f, 0.01f, 0.01f, 0.01f}, {0.97f}, 0.5f);
279
    test_typical({0.4f, 0.2f, 0.2f, 0.2f}, {0.2f, 0.2f, 0.2f}, 0.5f);
280


281
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0}, {0.25f, 0.25f, 0.25f, 0.25f, 0},   50.0f, 0.0f, 0.0f);
282
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2}, {0.5f, 0.5f, 0, 0, 0},       50.0f, 0.0f, 0.0f);
283
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 0, 0}, {0.5f, 0.5f, 0, 0, 0}, 50.0f, 0.0f, 0.0f);
284


285
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0},             {0.249997f, 0.249997f, 0.249997f, 0.249997f, 0.000011f}, 1.0f, 5.0f, 5.0f);
286
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2},       {0.499966f, 0.499966f, 0.000023f, 0.000023f, 0.000023f}, 1.0f, 5.0f, 5.0f);
287
    test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 0, 0}, {0.499977f, 0.499977f, 0.000023f, 0.000023f, 0.000000f}, 1.0f, 5.0f, 5.0f);
288


289
    test_sampler_queue(10000, "k", 10000, 1.0f, 1.0f);
290
    test_sampler_queue(10000, "k",     1, 1.0f, 1.0f);
291
    test_sampler_queue(10000, "p", 10000, 1.0f, 1.0f);
292
    test_sampler_queue(10000, "p", 10000, 0.0f, 1.0f);
293
    test_sampler_queue(10000, "m", 10000, 1.0f, 1.0f);
294
    test_sampler_queue(10000, "m", 10000, 1.0f, 1e-12);
295


296
    test_sampler_queue(10000, "k",   100, 1.0000f, 1.0f);
297
    test_sampler_queue(10000, "p", 10000, 0.0002f, 1.0f);
298
    test_sampler_queue(10000, "p", 10000, 0.8000f, 1.0f);
299
    test_sampler_queue(10000, "m", 10000, 1.0000f, 9997.9f/9999.0f);
300
    test_sampler_queue(10000, "m", 10000, 1.0000f, 0.1f);
301


302
    test_sampler_queue(10000, "kp", 100, 0.8f, 0.1f);
303
    test_sampler_queue(10000, "km", 100, 0.8f, 0.1f);
304
    test_sampler_queue(10000, "pk", 100, 0.8f, 0.1f);
305
    test_sampler_queue(10000, "pm", 100, 0.8f, 0.1f);
306
    test_sampler_queue(10000, "mk", 100, 0.8f, 0.1f);
307
    test_sampler_queue(10000, "mp", 100, 0.8f, 9997.9f/9999.0f);
308
    test_sampler_queue(10000, "mp", 100, 0.8f, 0.1f);
309


310
    test_sampler_queue(10000, "kpm", 100, 0.8f, 0.1f);
311
    test_sampler_queue(10000, "kmp", 100, 0.8f, 0.1f);
312
    test_sampler_queue(10000, "pkm", 100, 0.8f, 0.1f);
313
    test_sampler_queue(10000, "pmk", 100, 0.8f, 0.1f);
314
    test_sampler_queue(10000, "mkp", 100, 0.8f, 0.1f);
315
    test_sampler_queue(10000, "mpk", 100, 0.8f, 0.1f);
316


317
    printf("OK\n");
318


319
    return 0;
320
}
321

Использование cookies

Мы используем файлы cookie в соответствии с Политикой конфиденциальности и Политикой использования cookies.

Нажимая кнопку «Принимаю», Вы даете АО «СберТех» согласие на обработку Ваших персональных данных в целях совершенствования нашего веб-сайта и Сервиса GitVerse, а также повышения удобства их использования.

Запретить использование cookies Вы можете самостоятельно в настройках Вашего браузера.