llama

1
#include <algorithm>
2
#include <array>
3
#include <cassert>
4
#include <chrono>
5
#include <cinttypes>
6
#include <clocale>
7
#include <cmath>
8
#include <cstdio>
9
#include <cstring>
10
#include <ctime>
11
#include <cstdlib>
12
#include <iterator>
13
#include <map>
14
#include <numeric>
15
#include <regex>
16
#include <sstream>
17
#include <string>
18
#include <vector>
19
#include <thread>
20

21
#include "ggml.h"
22
#include "llama.h"
23
#include "common.h"
24
#include "ggml-cuda.h"
25
#include "ggml-sycl.h"
26

27
#ifdef GGML_USE_CANN
28
#include "ggml-cann.h"
29
#endif
30

31
#ifdef _WIN32
32
#define WIN32_LEAN_AND_MEAN
33
#ifndef NOMINMAX
34
#   define NOMINMAX
35
#endif
36
#include <windows.h>
37
#endif
38

39
// utils
40
static uint64_t get_time_ns() {
41
    using clock = std::chrono::high_resolution_clock;
42
    return std::chrono::nanoseconds(clock::now().time_since_epoch()).count();
43
}
44

45
template<class T>
46
static std::string join(const std::vector<T> & values, const std::string & delim) {
47
    std::ostringstream str;
48
    for (size_t i = 0; i < values.size(); i++) {
49
        str << values[i];
50
        if (i < values.size() - 1) {
51
            str << delim;
52
        }
53
    }
54
    return str.str();
55
}
56

57
template<typename T, typename F>
58
static std::vector<std::string> transform_to_str(const std::vector<T> & values, F f) {
59
    std::vector<std::string> str_values;
60
    std::transform(values.begin(), values.end(), std::back_inserter(str_values), f);
61
    return str_values;
62
}
63

64
template<typename T>
65
static T avg(const std::vector<T> & v) {
66
    if (v.empty()) {
67
        return 0;
68
    }
69
    T sum = std::accumulate(v.begin(), v.end(), T(0));
70
    return sum / (T)v.size();
71
}
72

73
template<typename T>
74
static T stdev(const std::vector<T> & v) {
75
    if (v.size() <= 1) {
76
        return 0;
77
    }
78
    T mean = avg(v);
79
    T sq_sum = std::inner_product(v.begin(), v.end(), v.begin(), T(0));
80
    T stdev = std::sqrt(sq_sum / (T)(v.size() - 1) - mean * mean * (T)v.size() / (T)(v.size() - 1));
81
    return stdev;
82
}
83

84
static std::string get_cpu_info() {
85
    std::string id;
86
#ifdef __linux__
87
    FILE * f = fopen("/proc/cpuinfo", "r");
88
    if (f) {
89
        char buf[1024];
90
        while (fgets(buf, sizeof(buf), f)) {
91
            if (strncmp(buf, "model name", 10) == 0) {
92
                char * p = strchr(buf, ':');
93
                if (p) {
94
                    p++;
95
                    while (std::isspace(*p)) {
96
                        p++;
97
                    }
98
                    while (std::isspace(p[strlen(p) - 1])) {
99
                        p[strlen(p) - 1] = '\0';
100
                    }
101
                    id = p;
102
                    break;
103
                }
104
            }
105
        }
106
        fclose(f);
107
    }
108
#elif defined(_WIN32)
109
    HKEY hKey;
110
    if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,
111
                     TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"),
112
                     0,
113
                     KEY_READ,
114
                     &hKey) != ERROR_SUCCESS) {
115
        // fail to open registry key
116
        return "";
117
    }
118
    char cpu_brand[256];
119
    DWORD cpu_brand_size = sizeof(cpu_brand);
120
    if (RegQueryValueExA(hKey,
121
                        TEXT("ProcessorNameString"),
122
                        NULL,
123
                        NULL,
124
                        (LPBYTE)cpu_brand,
125
                        &cpu_brand_size) == ERROR_SUCCESS) {
126
        id.assign(cpu_brand, cpu_brand_size);
127
        if (id.find('\0') != std::string::npos) {
128
            id.resize(id.find('\0'));
129
        }
130
    }
131
    RegCloseKey(hKey);
132
#endif
133
    // TODO: other platforms
134
    return id;
135
}
136

137
static std::string get_gpu_info() {
138
    std::string id;
139
#ifdef GGML_USE_CUDA
140
    int count = ggml_backend_cuda_get_device_count();
141
    for (int i = 0; i < count; i++) {
142
        char buf[128];
143
        ggml_backend_cuda_get_device_description(i, buf, sizeof(buf));
144
        id += buf;
145
        if (i < count - 1) {
146
            id += "/";
147
        }
148
    }
149
#endif
150
#ifdef GGML_USE_SYCL
151
    int count = ggml_backend_sycl_get_device_count();
152
    for (int i = 0; i < count; i++) {
153
        char buf[128];
154
        ggml_sycl_get_device_description(i, buf, sizeof(buf));
155
        id += buf;
156
        if (i < count - 1) {
157
            id += "/";
158
        }
159
    }
160
#endif
161
#ifdef GGML_USE_CANN
162
    uint32_t count = ggml_backend_cann_get_device_count();
163
    for (uint32_t i = 0; i < count; i++) {
164
        char buf[128];
165
        ggml_backend_cann_get_device_description(i, buf, sizeof(buf));
166
        id += buf;
167
        if (i < count - 1) {
168
            id += "/";
169
        }
170
    }
171
#endif
172
    // TODO: other backends
173
    return id;
174
}
175

176
// command line params
177
enum output_formats {NONE, CSV, JSON, JSONL, MARKDOWN, SQL};
178

179
static const char * output_format_str(output_formats format) {
180
    switch (format) {
181
        case NONE:     return "none";
182
        case CSV:      return "csv";
183
        case JSON:     return "json";
184
        case JSONL:    return "jsonl";
185
        case MARKDOWN: return "md";
186
        case SQL:      return "sql";
187
        default: GGML_ABORT("invalid output format");
188
    }
189
}
190

191
static bool output_format_from_str(const std::string & s, output_formats & format) {
192
    if (s == "none") {
193
        format = NONE;
194
    } else if (s == "csv") {
195
        format = CSV;
196
    } else if (s == "json") {
197
        format = JSON;
198
    } else if (s == "jsonl") {
199
        format = JSONL;
200
    } else if (s == "md") {
201
        format = MARKDOWN;
202
    } else if (s == "sql") {
203
        format = SQL;
204
    } else {
205
        return false;
206
    }
207
    return true;
208
}
209

210
static const char * split_mode_str(llama_split_mode mode) {
211
    switch (mode) {
212
        case LLAMA_SPLIT_MODE_NONE:  return "none";
213
        case LLAMA_SPLIT_MODE_LAYER: return "layer";
214
        case LLAMA_SPLIT_MODE_ROW:   return "row";
215
        default: GGML_ABORT("invalid split mode");
216
    }
217
}
218

219
static std::string pair_str(const std::pair<int, int> & p) {
220
    static char buf[32];
221
    snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);
222
    return buf;
223
}
224

225
struct cmd_params {
226
    std::vector<std::string> model;
227
    std::vector<int> n_prompt;
228
    std::vector<int> n_gen;
229
    std::vector<std::pair<int, int>> n_pg;
230
    std::vector<int> n_batch;
231
    std::vector<int> n_ubatch;
232
    std::vector<ggml_type> type_k;
233
    std::vector<ggml_type> type_v;
234
    std::vector<int> n_threads;
235
    std::vector<std::string> cpu_mask;
236
    std::vector<bool> cpu_strict;
237
    std::vector<int> poll;
238
    std::vector<int> n_gpu_layers;
239
    std::vector<std::string> rpc_servers;
240
    std::vector<llama_split_mode> split_mode;
241
    std::vector<int> main_gpu;
242
    std::vector<bool> no_kv_offload;
243
    std::vector<bool> flash_attn;
244
    std::vector<std::vector<float>> tensor_split;
245
    std::vector<bool> use_mmap;
246
    std::vector<bool> embeddings;
247
    ggml_numa_strategy numa;
248
    int reps;
249
    ggml_sched_priority prio;
250
    int delay;
251
    bool verbose;
252
    bool progress;
253
    output_formats output_format;
254
    output_formats output_format_stderr;
255
};
256

257
static const cmd_params cmd_params_defaults = {
258
    /* model                */ {"models/7B/ggml-model-q4_0.gguf"},
259
    /* n_prompt             */ {512},
260
    /* n_gen                */ {128},
261
    /* n_pg                 */ {},
262
    /* n_batch              */ {2048},
263
    /* n_ubatch             */ {512},
264
    /* type_k               */ {GGML_TYPE_F16},
265
    /* type_v               */ {GGML_TYPE_F16},
266
    /* n_threads            */ {cpu_get_num_math()},
267
    /* cpu_mask             */ {"0x0"},
268
    /* cpu_strict           */ {false},
269
    /* poll                 */ {50},
270
    /* n_gpu_layers         */ {99},
271
    /* rpc_servers          */ {""},
272
    /* split_mode           */ {LLAMA_SPLIT_MODE_LAYER},
273
    /* main_gpu             */ {0},
274
    /* no_kv_offload        */ {false},
275
    /* flash_attn           */ {false},
276
    /* tensor_split         */ {std::vector<float>(llama_max_devices(), 0.0f)},
277
    /* use_mmap             */ {true},
278
    /* embeddings           */ {false},
279
    /* numa                 */ GGML_NUMA_STRATEGY_DISABLED,
280
    /* reps                 */ 5,
281
    /* prio                 */ GGML_SCHED_PRIO_NORMAL,
282
    /* delay                */ 0,
283
    /* verbose              */ false,
284
    /* progress             */ false,
285
    /* output_format        */ MARKDOWN,
286
    /* output_format_stderr */ NONE,
287
};
288

289
static void print_usage(int /* argc */, char ** argv) {
290
    printf("usage: %s [options]\n", argv[0]);
291
    printf("\n");
292
    printf("options:\n");
293
    printf("  -h, --help\n");
294
    printf("  -m, --model <filename>                    (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());
295
    printf("  -p, --n-prompt <n>                        (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());
296
    printf("  -n, --n-gen <n>                           (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());
297
    printf("  -pg <pp,tg>                               (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());
298
    printf("  -b, --batch-size <n>                      (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());
299
    printf("  -ub, --ubatch-size <n>                    (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());
300
    printf("  -ctk, --cache-type-k <t>                  (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());
301
    printf("  -ctv, --cache-type-v <t>                  (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());
302
    printf("  -t, --threads <n>                         (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());
303
    printf("  -C, --cpu-mask <hex,hex>                  (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());
304
    printf("  --cpu-strict <0|1>                        (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());
305
    printf("  --poll <0...100>                          (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());
306
    printf("  -ngl, --n-gpu-layers <n>                  (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());
307
#ifdef GGML_USE_RPC
308
    printf("  -rpc, --rpc <rpc_servers>                 (default: %s)\n", join(cmd_params_defaults.rpc_servers, ",").c_str());
309
#endif
310
    printf("  -sm, --split-mode <none|layer|row>        (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
311
    printf("  -mg, --main-gpu <i>                       (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());
312
    printf("  -nkvo, --no-kv-offload <0|1>              (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
313
    printf("  -fa, --flash-attn <0|1>                   (default: %s)\n", join(cmd_params_defaults.flash_attn, ",").c_str());
314
    printf("  -mmp, --mmap <0|1>                        (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str());
315
    printf("  --numa <distribute|isolate|numactl>       (default: disabled)\n");
316
    printf("  -embd, --embeddings <0|1>                 (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
317
    printf("  -ts, --tensor-split <ts0/ts1/..>          (default: 0)\n");
318
    printf("  -r, --repetitions <n>                     (default: %d)\n", cmd_params_defaults.reps);
319
    printf("  --prio <0|1|2|3>                          (default: %d)\n", cmd_params_defaults.prio);
320
    printf("  --delay <0...N> (seconds)                 (default: %d)\n", cmd_params_defaults.delay);
321
    printf("  -o, --output <csv|json|jsonl|md|sql>      (default: %s)\n", output_format_str(cmd_params_defaults.output_format));
322
    printf("  -oe, --output-err <csv|json|jsonl|md|sql> (default: %s)\n", output_format_str(cmd_params_defaults.output_format_stderr));
323
    printf("  -v, --verbose                             (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0");
324
    printf("  --progress                                (default: %s)\n", cmd_params_defaults.progress ? "1" : "0");
325
    printf("\n");
326
    printf("Multiple values can be given for each parameter by separating them with ',' or by specifying the parameter multiple times.\n");
327
}
328

329
static ggml_type ggml_type_from_name(const std::string & s) {
330
    if (s == "f16") {
331
        return GGML_TYPE_F16;
332
    }
333
    if (s == "q8_0") {
334
        return GGML_TYPE_Q8_0;
335
    }
336
    if (s == "q4_0") {
337
        return GGML_TYPE_Q4_0;
338
    }
339
    if (s == "q4_1") {
340
        return GGML_TYPE_Q4_1;
341
    }
342
    if (s == "q5_0") {
343
        return GGML_TYPE_Q5_0;
344
    }
345
    if (s == "q5_1") {
346
        return GGML_TYPE_Q5_1;
347
    }
348
    if (s == "iq4_nl") {
349
        return GGML_TYPE_IQ4_NL;
350
    }
351

352
    return GGML_TYPE_COUNT;
353
}
354

355

356
static cmd_params parse_cmd_params(int argc, char ** argv) {
357
    cmd_params params;
358
    std::string arg;
359
    bool invalid_param = false;
360
    const std::string arg_prefix = "--";
361
    const char split_delim = ',';
362

363
    params.verbose = cmd_params_defaults.verbose;
364
    params.output_format = cmd_params_defaults.output_format;
365
    params.output_format_stderr = cmd_params_defaults.output_format_stderr;
366
    params.reps = cmd_params_defaults.reps;
367
    params.numa = cmd_params_defaults.numa;
368
    params.prio = cmd_params_defaults.prio;
369
    params.delay = cmd_params_defaults.delay;
370
    params.progress = cmd_params_defaults.progress;
371

372
    for (int i = 1; i < argc; i++) {
373
        arg = argv[i];
374
        if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
375
            std::replace(arg.begin(), arg.end(), '_', '-');
376
        }
377

378
        if (arg == "-h" || arg == "--help") {
379
            print_usage(argc, argv);
380
            exit(0);
381
        } else if (arg == "-m" || arg == "--model") {
382
            if (++i >= argc) {
383
                invalid_param = true;
384
                break;
385
            }
386
            auto p = string_split<std::string>(argv[i], split_delim);
387
            params.model.insert(params.model.end(), p.begin(), p.end());
388
        } else if (arg == "-p" || arg == "--n-prompt") {
389
            if (++i >= argc) {
390
                invalid_param = true;
391
                break;
392
            }
393
            auto p = string_split<int>(argv[i], split_delim);
394
            params.n_prompt.insert(params.n_prompt.end(), p.begin(), p.end());
395
        } else if (arg == "-n" || arg == "--n-gen") {
396
            if (++i >= argc) {
397
                invalid_param = true;
398
                break;
399
            }
400
            auto p = string_split<int>(argv[i], split_delim);
401
            params.n_gen.insert(params.n_gen.end(), p.begin(), p.end());
402
        } else if (arg == "-pg") {
403
            if (++i >= argc) {
404
                invalid_param = true;
405
                break;
406
            }
407
            auto p = string_split<std::string>(argv[i], ',');
408
            if (p.size() != 2) {
409
                invalid_param = true;
410
                break;
411
            }
412
            params.n_pg.push_back({std::stoi(p[0]), std::stoi(p[1])});
413
        } else if (arg == "-b" || arg == "--batch-size") {
414
            if (++i >= argc) {
415
                invalid_param = true;
416
                break;
417
            }
418
            auto p = string_split<int>(argv[i], split_delim);
419
            params.n_batch.insert(params.n_batch.end(), p.begin(), p.end());
420
        } else if (arg == "-ub" || arg == "--ubatch-size") {
421
            if (++i >= argc) {
422
                invalid_param = true;
423
                break;
424
            }
425
            auto p = string_split<int>(argv[i], split_delim);
426
            params.n_ubatch.insert(params.n_ubatch.end(), p.begin(), p.end());
427
        } else if (arg == "-ctk" || arg == "--cache-type-k") {
428
            if (++i >= argc) {
429
                invalid_param = true;
430
                break;
431
            }
432
            auto p = string_split<std::string>(argv[i], split_delim);
433
            std::vector<ggml_type> types;
434
            for (const auto & t : p) {
435
                ggml_type gt = ggml_type_from_name(t);
436
                if (gt == GGML_TYPE_COUNT) {
437
                    invalid_param = true;
438
                    break;
439
                }
440
                types.push_back(gt);
441
            }
442
            if (invalid_param) {
443
                break;
444
            }
445
            params.type_k.insert(params.type_k.end(), types.begin(), types.end());
446
        } else if (arg == "-ctv" || arg == "--cache-type-v") {
447
            if (++i >= argc) {
448
                invalid_param = true;
449
                break;
450
            }
451
            auto p = string_split<std::string>(argv[i], split_delim);
452
            std::vector<ggml_type> types;
453
            for (const auto & t : p) {
454
                ggml_type gt = ggml_type_from_name(t);
455
                if (gt == GGML_TYPE_COUNT) {
456
                    invalid_param = true;
457
                    break;
458
                }
459
                types.push_back(gt);
460
            }
461
            if (invalid_param) {
462
                break;
463
            }
464
            params.type_v.insert(params.type_v.end(), types.begin(), types.end());
465
        } else if (arg == "-t" || arg == "--threads") {
466
            if (++i >= argc) {
467
                invalid_param = true;
468
                break;
469
            }
470
            auto p = string_split<int>(argv[i], split_delim);
471
            params.n_threads.insert(params.n_threads.end(), p.begin(), p.end());
472
        } else if (arg == "-C" || arg == "--cpu-mask") {
473
            if (++i >= argc) {
474
                invalid_param = true;
475
                break;
476
            }
477
            auto p = string_split<std::string>(argv[i], split_delim);
478
            params.cpu_mask.insert(params.cpu_mask.end(), p.begin(), p.end());
479
        } else if (arg == "--cpu-strict") {
480
            if (++i >= argc) {
481
                invalid_param = true;
482
                break;
483
            }
484
            auto p = string_split<bool>(argv[i], split_delim);
485
            params.cpu_strict.insert(params.cpu_strict.end(), p.begin(), p.end());
486
        } else if (arg == "--poll") {
487
            if (++i >= argc) {
488
                invalid_param = true;
489
                break;
490
            }
491
            auto p = string_split<int>(argv[i], split_delim);
492
            params.poll.insert(params.poll.end(), p.begin(), p.end());
493
        } else if (arg == "-ngl" || arg == "--n-gpu-layers") {
494
            if (++i >= argc) {
495
                invalid_param = true;
496
                break;
497
            }
498
            auto p = string_split<int>(argv[i], split_delim);
499
            params.n_gpu_layers.insert(params.n_gpu_layers.end(), p.begin(), p.end());
500
#ifdef GGML_USE_RPC
501
        } else if (arg == "-rpc" || arg == "--rpc") {
502
            if (++i >= argc) {
503
                invalid_param = true;
504
                break;
505
            }
506
            params.rpc_servers.push_back(argv[i]);
507
#endif
508
        } else if (arg == "-sm" || arg == "--split-mode") {
509
            if (++i >= argc) {
510
                invalid_param = true;
511
                break;
512
            }
513
            auto p = string_split<std::string>(argv[i], split_delim);
514
            std::vector<llama_split_mode> modes;
515
            for (const auto & m : p) {
516
                llama_split_mode mode;
517
                if (m == "none") {
518
                    mode = LLAMA_SPLIT_MODE_NONE;
519
                } else if (m == "layer") {
520
                    mode = LLAMA_SPLIT_MODE_LAYER;
521
                } else if (m == "row") {
522
                    mode = LLAMA_SPLIT_MODE_ROW;
523
                } else {
524
                    invalid_param = true;
525
                    break;
526
                }
527
                modes.push_back(mode);
528
            }
529
            if (invalid_param) {
530
                break;
531
            }
532
            params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end());
533
        } else if (arg == "-mg" || arg == "--main-gpu") {
534
            if (++i >= argc) {
535
                invalid_param = true;
536
                break;
537
            }
538
            params.main_gpu = string_split<int>(argv[i], split_delim);
539
        } else if (arg == "-nkvo" || arg == "--no-kv-offload") {
540
            if (++i >= argc) {
541
                invalid_param = true;
542
                break;
543
            }
544
            auto p = string_split<bool>(argv[i], split_delim);
545
            params.no_kv_offload.insert(params.no_kv_offload.end(), p.begin(), p.end());
546
        } else if (arg == "--numa") {
547
            if (++i >= argc) {
548
                invalid_param = true;
549
                break;
550
            } else {
551
                std::string value(argv[i]);
552
                /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }
553
                else if (value == "isolate")                    { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
554
                else if (value == "numactl")                    { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
555
                else { invalid_param = true; break; }
556
            }
557
        } else if (arg == "-fa" || arg == "--flash-attn") {
558
            if (++i >= argc) {
559
                invalid_param = true;
560
                break;
561
            }
562
            auto p = string_split<bool>(argv[i], split_delim);
563
            params.flash_attn.insert(params.flash_attn.end(), p.begin(), p.end());
564
        } else if (arg == "-mmp" || arg == "--mmap") {
565
            if (++i >= argc) {
566
                invalid_param = true;
567
                break;
568
            }
569
            auto p = string_split<bool>(argv[i], split_delim);
570
            params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end());
571
        } else if (arg == "-embd" || arg == "--embeddings") {
572
            if (++i >= argc) {
573
                invalid_param = true;
574
                break;
575
            }
576
            auto p = string_split<bool>(argv[i], split_delim);
577
            params.embeddings.insert(params.embeddings.end(), p.begin(), p.end());
578
        } else if (arg == "-ts" || arg == "--tensor-split") {
579
            if (++i >= argc) {
580
                invalid_param = true;
581
                break;
582
            }
583
            for (auto ts : string_split<std::string>(argv[i], split_delim)) {
584
                // split string by ; and /
585
                const std::regex regex{R"([;/]+)"};
586
                std::sregex_token_iterator it{ts.begin(), ts.end(), regex, -1};
587
                std::vector<std::string> split_arg{it, {}};
588
                GGML_ASSERT(split_arg.size() <= llama_max_devices());
589

590
                std::vector<float> tensor_split(llama_max_devices());
591
                for (size_t i = 0; i < llama_max_devices(); ++i) {
592
                    if (i < split_arg.size()) {
593
                        tensor_split[i] = std::stof(split_arg[i]);
594
                    } else {
595
                        tensor_split[i] = 0.0f;
596
                    }
597
                }
598
                params.tensor_split.push_back(tensor_split);
599
            }
600
        } else if (arg == "-r" || arg == "--repetitions") {
601
            if (++i >= argc) {
602
                invalid_param = true;
603
                break;
604
            }
605
            params.reps = std::stoi(argv[i]);
606
        } else if (arg == "--prio") {
607
            if (++i >= argc) {
608
                invalid_param = true;
609
                break;
610
            }
611
            params.prio = (enum ggml_sched_priority) std::stoi(argv[i]);
612
        } else if (arg == "--delay") {
613
            if (++i >= argc) {
614
                invalid_param = true;
615
                break;
616
            }
617
            params.delay = std::stoi(argv[i]);
618
        } else if (arg == "-o" || arg == "--output") {
619
            if (++i >= argc) {
620
                invalid_param = true;
621
                break;
622
            }
623
            invalid_param = !output_format_from_str(argv[i], params.output_format);
624
        } else if (arg == "-oe" || arg == "--output-err") {
625
            if (++i >= argc) {
626
                invalid_param = true;
627
                break;
628
            }
629
            invalid_param = !output_format_from_str(argv[i], params.output_format_stderr);
630
        } else if (arg == "-v" || arg == "--verbose") {
631
            params.verbose = true;
632
        } else if (arg == "--progress") {
633
            params.progress = true;
634
        } else {
635
            invalid_param = true;
636
            break;
637
        }
638
    }
639
    if (invalid_param) {
640
        fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
641
        print_usage(argc, argv);
642
        exit(1);
643
    }
644

645
    // set defaults
646
    if (params.model.empty())        { params.model = cmd_params_defaults.model; }
647
    if (params.n_prompt.empty())     { params.n_prompt = cmd_params_defaults.n_prompt; }
648
    if (params.n_gen.empty())        { params.n_gen = cmd_params_defaults.n_gen; }
649
    if (params.n_pg.empty())         { params.n_pg = cmd_params_defaults.n_pg; }
650
    if (params.n_batch.empty())      { params.n_batch = cmd_params_defaults.n_batch; }
651
    if (params.n_ubatch.empty())     { params.n_ubatch = cmd_params_defaults.n_ubatch; }
652
    if (params.type_k.empty())       { params.type_k = cmd_params_defaults.type_k; }
653
    if (params.type_v.empty())       { params.type_v = cmd_params_defaults.type_v; }
654
    if (params.n_gpu_layers.empty()) { params.n_gpu_layers = cmd_params_defaults.n_gpu_layers; }
655
    if (params.rpc_servers.empty())  { params.rpc_servers = cmd_params_defaults.rpc_servers; }
656
    if (params.split_mode.empty())   { params.split_mode = cmd_params_defaults.split_mode; }
657
    if (params.main_gpu.empty())     { params.main_gpu = cmd_params_defaults.main_gpu; }
658
    if (params.no_kv_offload.empty()){ params.no_kv_offload = cmd_params_defaults.no_kv_offload; }
659
    if (params.flash_attn.empty())   { params.flash_attn = cmd_params_defaults.flash_attn; }
660
    if (params.tensor_split.empty()) { params.tensor_split = cmd_params_defaults.tensor_split; }
661
    if (params.use_mmap.empty())     { params.use_mmap = cmd_params_defaults.use_mmap; }
662
    if (params.embeddings.empty())   { params.embeddings = cmd_params_defaults.embeddings; }
663
    if (params.n_threads.empty())    { params.n_threads = cmd_params_defaults.n_threads; }
664
    if (params.cpu_mask.empty())     { params.cpu_mask  = cmd_params_defaults.cpu_mask;  }
665
    if (params.cpu_strict.empty())   { params.cpu_strict = cmd_params_defaults.cpu_strict; }
666
    if (params.poll.empty())         { params.poll = cmd_params_defaults.poll; }
667

668
    return params;
669
}
670

671
struct cmd_params_instance {
672
    std::string model;
673
    int n_prompt;
674
    int n_gen;
675
    int n_batch;
676
    int n_ubatch;
677
    ggml_type type_k;
678
    ggml_type type_v;
679
    int n_threads;
680
    std::string cpu_mask;
681
    bool cpu_strict;
682
    int poll;
683
    int n_gpu_layers;
684
    std::string rpc_servers;
685
    llama_split_mode split_mode;
686
    int main_gpu;
687
    bool no_kv_offload;
688
    bool flash_attn;
689
    std::vector<float> tensor_split;
690
    bool use_mmap;
691
    bool embeddings;
692

693
    llama_model_params to_llama_mparams() const {
694
        llama_model_params mparams = llama_model_default_params();
695

696
        mparams.n_gpu_layers = n_gpu_layers;
697
        if (!rpc_servers.empty()) {
698
            mparams.rpc_servers = rpc_servers.c_str();
699
        }
700
        mparams.split_mode = split_mode;
701
        mparams.main_gpu = main_gpu;
702
        mparams.tensor_split = tensor_split.data();
703
        mparams.use_mmap = use_mmap;
704

705
        return mparams;
706
    }
707

708
    bool equal_mparams(const cmd_params_instance & other) const {
709
        return model == other.model &&
710
               n_gpu_layers == other.n_gpu_layers &&
711
               rpc_servers == other.rpc_servers &&
712
               split_mode == other.split_mode &&
713
               main_gpu == other.main_gpu &&
714
               use_mmap == other.use_mmap &&
715
               tensor_split == other.tensor_split;
716
    }
717

718
    llama_context_params to_llama_cparams() const {
719
        llama_context_params cparams = llama_context_default_params();
720

721
        cparams.n_ctx = n_prompt + n_gen;
722
        cparams.n_batch = n_batch;
723
        cparams.n_ubatch = n_ubatch;
724
        cparams.type_k = type_k;
725
        cparams.type_v = type_v;
726
        cparams.offload_kqv = !no_kv_offload;
727
        cparams.flash_attn = flash_attn;
728
        cparams.embeddings = embeddings;
729

730
        return cparams;
731
    }
732
};
733

734
static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_params & params) {
735
    std::vector<cmd_params_instance> instances;
736

737
    // this ordering minimizes the number of times that each model needs to be reloaded
738
    for (const auto & m : params.model)
739
    for (const auto & nl : params.n_gpu_layers)
740
    for (const auto & rpc : params.rpc_servers)
741
    for (const auto & sm : params.split_mode)
742
    for (const auto & mg : params.main_gpu)
743
    for (const auto & ts : params.tensor_split)
744
    for (const auto & mmp : params.use_mmap)
745
    for (const auto & embd : params.embeddings)
746
    for (const auto & nb : params.n_batch)
747
    for (const auto & nub : params.n_ubatch)
748
    for (const auto & tk : params.type_k)
749
    for (const auto & tv : params.type_v)
750
    for (const auto & nkvo : params.no_kv_offload)
751
    for (const auto & fa : params.flash_attn)
752
    for (const auto & nt : params.n_threads)
753
    for (const auto & cm : params.cpu_mask)
754
    for (const auto & cs : params.cpu_strict)
755
    for (const auto & pl : params.poll) {
756
        for (const auto & n_prompt : params.n_prompt) {
757
            if (n_prompt == 0) {
758
                continue;
759
            }
760
            cmd_params_instance instance = {
761
                /* .model        = */ m,
762
                /* .n_prompt     = */ n_prompt,
763
                /* .n_gen        = */ 0,
764
                /* .n_batch      = */ nb,
765
                /* .n_ubatch     = */ nub,
766
                /* .type_k       = */ tk,
767
                /* .type_v       = */ tv,
768
                /* .n_threads    = */ nt,
769
                /* .cpu_mask     = */ cm,
770
                /* .cpu_strict   = */ cs,
771
                /* .poll         = */ pl,
772
                /* .n_gpu_layers = */ nl,
773
                /* .rpc_servers  = */ rpc,
774
                /* .split_mode   = */ sm,
775
                /* .main_gpu     = */ mg,
776
                /* .no_kv_offload= */ nkvo,
777
                /* .flash_attn   = */ fa,
778
                /* .tensor_split = */ ts,
779
                /* .use_mmap     = */ mmp,
780
                /* .embeddings   = */ embd,
781
            };
782
            instances.push_back(instance);
783
        }
784

785
        for (const auto & n_gen : params.n_gen) {
786
            if (n_gen == 0) {
787
                continue;
788
            }
789
            cmd_params_instance instance = {
790
                /* .model        = */ m,
791
                /* .n_prompt     = */ 0,
792
                /* .n_gen        = */ n_gen,
793
                /* .n_batch      = */ nb,
794
                /* .n_ubatch     = */ nub,
795
                /* .type_k       = */ tk,
796
                /* .type_v       = */ tv,
797
                /* .n_threads    = */ nt,
798
                /* .cpu_mask     = */ cm,
799
                /* .cpu_strict   = */ cs,
800
                /* .poll         = */ pl,
801
                /* .n_gpu_layers = */ nl,
802
                /* .rpc_servers  = */ rpc,
803
                /* .split_mode   = */ sm,
804
                /* .main_gpu     = */ mg,
805
                /* .no_kv_offload= */ nkvo,
806
                /* .flash_attn   = */ fa,
807
                /* .tensor_split = */ ts,
808
                /* .use_mmap     = */ mmp,
809
                /* .embeddings   = */ embd,
810
            };
811
            instances.push_back(instance);
812
        }
813

814
        for (const auto & n_pg : params.n_pg) {
815
            if (n_pg.first == 0 && n_pg.second == 0) {
816
                continue;
817
            }
818
            cmd_params_instance instance = {
819
                /* .model        = */ m,
820
                /* .n_prompt     = */ n_pg.first,
821
                /* .n_gen        = */ n_pg.second,
822
                /* .n_batch      = */ nb,
823
                /* .n_ubatch     = */ nub,
824
                /* .type_k       = */ tk,
825
                /* .type_v       = */ tv,
826
                /* .n_threads    = */ nt,
827
                /* .cpu_mask     = */ cm,
828
                /* .cpu_strict   = */ cs,
829
                /* .poll         = */ pl,
830
                /* .n_gpu_layers = */ nl,
831
                /* .rpc_servers  = */ rpc,
832
                /* .split_mode   = */ sm,
833
                /* .main_gpu     = */ mg,
834
                /* .no_kv_offload= */ nkvo,
835
                /* .flash_attn   = */ fa,
836
                /* .tensor_split = */ ts,
837
                /* .use_mmap     = */ mmp,
838
                /* .embeddings   = */ embd,
839
            };
840
            instances.push_back(instance);
841
        }
842
    }
843

844
    return instances;
845
}
846

847
struct test {
848
    static const std::string build_commit;
849
    static const int build_number;
850
    static const bool cuda;
851
    static const bool vulkan;
852
    static const bool kompute;
853
    static const bool metal;
854
    static const bool sycl;
855
    static const bool gpu_blas;
856
    static const bool blas;
857
    static const std::string cpu_info;
858
    static const std::string gpu_info;
859
    std::string model_filename;
860
    std::string model_type;
861
    uint64_t model_size;
862
    uint64_t model_n_params;
863
    int n_batch;
864
    int n_ubatch;
865
    int n_threads;
866
    std::string cpu_mask;
867
    bool cpu_strict;
868
    int poll;
869
    bool has_rpc;
870
    ggml_type type_k;
871
    ggml_type type_v;
872
    int n_gpu_layers;
873
    llama_split_mode split_mode;
874
    int main_gpu;
875
    bool no_kv_offload;
876
    bool flash_attn;
877
    std::vector<float> tensor_split;
878
    bool use_mmap;
879
    bool embeddings;
880
    int n_prompt;
881
    int n_gen;
882
    std::string test_time;
883
    std::vector<uint64_t> samples_ns;
884

885
    test(const cmd_params_instance & inst, const llama_model * lmodel, const llama_context * ctx) {
886
        model_filename = inst.model;
887
        char buf[128];
888
        llama_model_desc(lmodel, buf, sizeof(buf));
889
        model_type = buf;
890
        model_size = llama_model_size(lmodel);
891
        model_n_params = llama_model_n_params(lmodel);
892
        n_batch = inst.n_batch;
893
        n_ubatch = inst.n_ubatch;
894
        n_threads = inst.n_threads;
895
        cpu_mask = inst.cpu_mask;
896
        cpu_strict = inst.cpu_strict;
897
        poll = inst.poll;
898
        has_rpc = !inst.rpc_servers.empty();
899
        type_k = inst.type_k;
900
        type_v = inst.type_v;
901
        n_gpu_layers = inst.n_gpu_layers;
902
        split_mode = inst.split_mode;
903
        main_gpu = inst.main_gpu;
904
        no_kv_offload = inst.no_kv_offload;
905
        flash_attn = inst.flash_attn;
906
        tensor_split = inst.tensor_split;
907
        use_mmap = inst.use_mmap;
908
        embeddings = inst.embeddings;
909
        n_prompt = inst.n_prompt;
910
        n_gen = inst.n_gen;
911
        // RFC 3339 date-time format
912
        time_t t = time(NULL);
913
        std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));
914
        test_time = buf;
915

916
        (void) ctx;
917
    }
918

919
    uint64_t avg_ns() const {
920
        return ::avg(samples_ns);
921
    }
922

923
    uint64_t stdev_ns() const {
924
        return ::stdev(samples_ns);
925
    }
926

927
    std::vector<double> get_ts() const {
928
        int n_tokens = n_prompt + n_gen;
929
        std::vector<double> ts;
930
        std::transform(samples_ns.begin(), samples_ns.end(), std::back_inserter(ts), [n_tokens](uint64_t t) { return 1e9 * n_tokens / t; });
931
        return ts;
932
    }
933

934
    double avg_ts() const {
935
        return ::avg(get_ts());
936
    }
937

938
    double stdev_ts() const {
939
        return ::stdev(get_ts());
940
    }
941

942
    static std::string get_backend() {
943
        if (cuda) {
944
            return GGML_CUDA_NAME;
945
        }
946
        if (vulkan) {
947
            return "Vulkan";
948
        }
949
        if (kompute) {
950
            return "Kompute";
951
        }
952
        if (metal) {
953
            return "Metal";
954
        }
955
        if (sycl) {
956
            return GGML_SYCL_NAME;
957
        }
958
        if (gpu_blas) {
959
            return "GPU BLAS";
960
        }
961
        if (blas) {
962
            return "BLAS";
963
        }
964

965
        return "CPU";
966
    }
967

968
    static const std::vector<std::string> & get_fields() {
969
        static const std::vector<std::string> fields = {
970
            "build_commit", "build_number",
971
            "cuda", "vulkan", "kompute", "metal", "sycl", "rpc", "gpu_blas", "blas",
972
            "cpu_info", "gpu_info",
973
            "model_filename", "model_type", "model_size", "model_n_params",
974
            "n_batch", "n_ubatch",
975
            "n_threads", "cpu_mask", "cpu_strict", "poll",
976
            "type_k", "type_v",
977
            "n_gpu_layers", "split_mode",
978
            "main_gpu", "no_kv_offload", "flash_attn",
979
            "tensor_split", "use_mmap", "embeddings",
980
            "n_prompt", "n_gen", "test_time",
981
            "avg_ns", "stddev_ns",
982
            "avg_ts", "stddev_ts",
983
        };
984
        return fields;
985
    }
986

987
    enum field_type {STRING, BOOL, INT, FLOAT};
988

989
    static field_type get_field_type(const std::string & field) {
990
        if (field == "build_number" || field == "n_batch" || field == "n_ubatch" ||
991
            field == "n_threads" || field == "poll" ||
992
            field == "model_size" || field == "model_n_params" ||
993
            field == "n_gpu_layers" || field == "main_gpu" ||
994
            field == "n_prompt" || field == "n_gen" ||
995
            field == "avg_ns" || field == "stddev_ns") {
996
            return INT;
997
        }
998
        if (field == "cuda" || field == "vulkan" || field == "kompute" || field == "metal" ||
999
            field == "gpu_blas" || field == "blas" || field == "sycl" ||field == "f16_kv" || field == "no_kv_offload" ||
1000
            field == "cpu_strict" ||
1001
            field == "flash_attn" || field == "use_mmap" || field == "embeddings") {
1002
            return BOOL;
1003
        }
1004
        if (field == "avg_ts" || field == "stddev_ts") {
1005
            return FLOAT;
1006
        }
1007
        return STRING;
1008
    }
1009

1010
    std::vector<std::string> get_values() const {
1011
        std::string tensor_split_str;
1012
        int max_nonzero = 0;
1013
        for (size_t i = 0; i < llama_max_devices(); i++) {
1014
            if (tensor_split[i] > 0) {
1015
                max_nonzero = i;
1016
            }
1017
        }
1018
        for (int i = 0; i <= max_nonzero; i++) {
1019
            char buf[32];
1020
            snprintf(buf, sizeof(buf), "%.2f", tensor_split[i]);
1021
            tensor_split_str += buf;
1022
            if (i < max_nonzero) {
1023
                tensor_split_str += "/";
1024
            }
1025
        }
1026
        std::vector<std::string> values = {
1027
            build_commit, std::to_string(build_number),
1028
            std::to_string(cuda), std::to_string(vulkan), std::to_string(vulkan),
1029
            std::to_string(metal), std::to_string(sycl), std::to_string(has_rpc), std::to_string(gpu_blas), std::to_string(blas),
1030
            cpu_info, gpu_info,
1031
            model_filename, model_type, std::to_string(model_size), std::to_string(model_n_params),
1032
            std::to_string(n_batch), std::to_string(n_ubatch),
1033
            std::to_string(n_threads), cpu_mask, std::to_string(cpu_strict), std::to_string(poll),
1034
            ggml_type_name(type_k), ggml_type_name(type_v),
1035
            std::to_string(n_gpu_layers), split_mode_str(split_mode),
1036
            std::to_string(main_gpu), std::to_string(no_kv_offload), std::to_string(flash_attn),
1037
            tensor_split_str, std::to_string(use_mmap), std::to_string(embeddings),
1038
            std::to_string(n_prompt), std::to_string(n_gen), test_time,
1039
            std::to_string(avg_ns()), std::to_string(stdev_ns()),
1040
            std::to_string(avg_ts()), std::to_string(stdev_ts())
1041
        };
1042
        return values;
1043
    }
1044

1045
    std::map<std::string, std::string> get_map() const {
1046
        std::map<std::string, std::string> map;
1047
        auto fields = get_fields();
1048
        auto values = get_values();
1049
        std::transform(fields.begin(), fields.end(), values.begin(),
1050
                std::inserter(map, map.end()), std::make_pair<const std::string &, const std::string &>);
1051
        return map;
1052
    }
1053
};
1054

1055
const std::string test::build_commit = LLAMA_COMMIT;
1056
const int         test::build_number = LLAMA_BUILD_NUMBER;
1057
const bool        test::cuda         = !!ggml_cpu_has_cuda();
1058
const bool        test::vulkan       = !!ggml_cpu_has_vulkan();
1059
const bool        test::kompute      = !!ggml_cpu_has_kompute();
1060
const bool        test::metal        = !!ggml_cpu_has_metal();
1061
const bool        test::gpu_blas     = !!ggml_cpu_has_gpublas();
1062
const bool        test::blas         = !!ggml_cpu_has_blas();
1063
const bool        test::sycl         = !!ggml_cpu_has_sycl();
1064
const std::string test::cpu_info     = get_cpu_info();
1065
const std::string test::gpu_info     = get_gpu_info();
1066

1067
struct printer {
1068
    virtual ~printer() {}
1069

1070
    FILE * fout;
1071
    virtual void print_header(const cmd_params & params) { (void) params; }
1072
    virtual void print_test(const test & t) = 0;
1073
    virtual void print_footer() { }
1074
};
1075

1076
struct csv_printer : public printer {
1077
    static std::string escape_csv(const std::string & field) {
1078
        std::string escaped = "\"";
1079
        for (auto c : field) {
1080
            if (c == '"') {
1081
                escaped += "\"";
1082
            }
1083
            escaped += c;
1084
        }
1085
        escaped += "\"";
1086
        return escaped;
1087
    }
1088

1089
    void print_header(const cmd_params & params) override  {
1090
        std::vector<std::string> fields = test::get_fields();
1091
        fprintf(fout, "%s\n", join(fields, ",").c_str());
1092
        (void) params;
1093
    }
1094

1095
    void print_test(const test & t) override {
1096
        std::vector<std::string> values = t.get_values();
1097
        std::transform(values.begin(), values.end(), values.begin(), escape_csv);
1098
        fprintf(fout, "%s\n", join(values, ",").c_str());
1099
    }
1100
};
1101

1102

1103
static std::string escape_json(const std::string & value) {
1104
    std::string escaped;
1105
    for (auto c : value) {
1106
        if (c == '"') {
1107
            escaped += "\\\"";
1108
        } else if (c == '\\') {
1109
            escaped += "\\\\";
1110
        } else  if (c <= 0x1f) {
1111
            char buf[8];
1112
            snprintf(buf, sizeof(buf), "\\u%04x", c);
1113
            escaped += buf;
1114
        } else {
1115
            escaped += c;
1116
        }
1117
    }
1118
    return escaped;
1119
}
1120

1121
static std::string format_json_value(const std::string & field, const std::string & value) {
1122
    switch (test::get_field_type(field)) {
1123
        case test::STRING:
1124
            return "\"" + escape_json(value) + "\"";
1125
        case test::BOOL:
1126
            return value == "0" ? "false" : "true";
1127
        default:
1128
            return value;
1129
    }
1130
}
1131

1132
struct json_printer : public printer {
1133
    bool first = true;
1134

1135
    void print_header(const cmd_params & params) override {
1136
        fprintf(fout, "[\n");
1137
        (void) params;
1138
    }
1139

1140
    void print_fields(const std::vector<std::string> & fields, const std::vector<std::string> & values) {
1141
        assert(fields.size() == values.size());
1142
        for (size_t i = 0; i < fields.size(); i++) {
1143
            fprintf(fout, "    \"%s\": %s,\n", fields.at(i).c_str(), format_json_value(fields.at(i), values.at(i)).c_str());
1144
        }
1145
    }
1146

1147
    void print_test(const test & t) override {
1148
        if (first) {
1149
            first = false;
1150
        } else {
1151
            fprintf(fout, ",\n");
1152
        }
1153
        fprintf(fout, "  {\n");
1154
        print_fields(test::get_fields(), t.get_values());
1155
        fprintf(fout, "    \"samples_ns\": [ %s ],\n", join(t.samples_ns, ", ").c_str());
1156
        fprintf(fout, "    \"samples_ts\": [ %s ]\n", join(t.get_ts(), ", ").c_str());
1157
        fprintf(fout, "  }");
1158
        fflush(fout);
1159
    }
1160

1161
    void print_footer() override {
1162
        fprintf(fout, "\n]\n");
1163
    }
1164
};
1165

1166

1167
struct jsonl_printer : public printer {
1168
    void print_fields(const std::vector<std::string> & fields, const std::vector<std::string> & values) {
1169
        assert(fields.size() == values.size());
1170
        for (size_t i = 0; i < fields.size(); i++) {
1171
            fprintf(fout, "\"%s\": %s, ", fields.at(i).c_str(), format_json_value(fields.at(i), values.at(i)).c_str());
1172
        }
1173
    }
1174

1175
    void print_test(const test & t) override {
1176
        fprintf(fout, "{");
1177
        print_fields(test::get_fields(), t.get_values());
1178
        fprintf(fout, "\"samples_ns\": [ %s ],", join(t.samples_ns, ", ").c_str());
1179
        fprintf(fout, "\"samples_ts\": [ %s ]", join(t.get_ts(), ", ").c_str());
1180
        fprintf(fout, "}\n");
1181
        fflush(fout);
1182
    }
1183
};
1184

1185
struct markdown_printer : public printer {
1186
    std::vector<std::string> fields;
1187

1188
    static int get_field_width(const std::string & field) {
1189
        if (field == "model") {
1190
            return -30;
1191
        }
1192
        if (field == "t/s") {
1193
            return 20;
1194
        }
1195
        if (field == "size" || field == "params") {
1196
            return 10;
1197
        }
1198
        if (field == "n_gpu_layers") {
1199
            return 3;
1200
        }
1201
        if (field == "n_threads") {
1202
            return 7;
1203
        }
1204
        if (field == "n_batch") {
1205
            return 7;
1206
        }
1207
        if (field == "n_ubatch") {
1208
            return 8;
1209
        }
1210
        if (field == "type_k" || field == "type_v") {
1211
            return 6;
1212
        }
1213
        if (field == "split_mode") {
1214
            return 5;
1215
        }
1216
        if (field == "flash_attn") {
1217
            return 2;
1218
        }
1219
        if (field == "use_mmap") {
1220
            return 4;
1221
        }
1222
        if (field == "test") {
1223
            return 13;
1224
        }
1225

1226
        int width = std::max((int)field.length(), 10);
1227

1228
        if (test::get_field_type(field) == test::STRING) {
1229
            return -width;
1230
        }
1231
        return width;
1232
    }
1233

1234
    static std::string get_field_display_name(const std::string & field) {
1235
        if (field == "n_gpu_layers") {
1236
            return "ngl";
1237
        }
1238
        if (field == "split_mode") {
1239
            return "sm";
1240
        }
1241
        if (field == "n_threads") {
1242
            return "threads";
1243
        }
1244
        if (field == "no_kv_offload") {
1245
            return "nkvo";
1246
        }
1247
        if (field == "flash_attn") {
1248
            return "fa";
1249
        }
1250
        if (field == "use_mmap") {
1251
            return "mmap";
1252
        }
1253
        if (field == "embeddings") {
1254
            return "embd";
1255
        }
1256
        if (field == "tensor_split") {
1257
            return "ts";
1258
        }
1259
        return field;
1260
    }
1261

1262
    void print_header(const cmd_params & params) override {
1263
        // select fields to print
1264
        fields.emplace_back("model");
1265
        fields.emplace_back("size");
1266
        fields.emplace_back("params");
1267
        fields.emplace_back("backend");
1268
        bool is_cpu_backend = test::get_backend() == "CPU" || test::get_backend() == "BLAS";
1269
        if (!is_cpu_backend) {
1270
            fields.emplace_back("n_gpu_layers");
1271
        }
1272
        if (params.n_threads.size() > 1 || params.n_threads != cmd_params_defaults.n_threads || is_cpu_backend) {
1273
            fields.emplace_back("n_threads");
1274
        }
1275
        if (params.cpu_mask.size() > 1 || params.cpu_mask != cmd_params_defaults.cpu_mask) {
1276
            fields.emplace_back("cpu_mask");
1277
        }
1278
        if (params.cpu_strict.size() > 1 || params.cpu_strict != cmd_params_defaults.cpu_strict) {
1279
            fields.emplace_back("cpu_strict");
1280
        }
1281
        if (params.poll.size() > 1 || params.poll != cmd_params_defaults.poll) {
1282
            fields.emplace_back("poll");
1283
        }
1284
        if (params.n_batch.size() > 1 || params.n_batch != cmd_params_defaults.n_batch) {
1285
            fields.emplace_back("n_batch");
1286
        }
1287
        if (params.n_ubatch.size() > 1 || params.n_ubatch != cmd_params_defaults.n_ubatch) {
1288
            fields.emplace_back("n_ubatch");
1289
        }
1290
        if (params.type_k.size() > 1 || params.type_k != cmd_params_defaults.type_k) {
1291
            fields.emplace_back("type_k");
1292
        }
1293
        if (params.type_v.size() > 1 || params.type_v != cmd_params_defaults.type_v) {
1294
            fields.emplace_back("type_v");
1295
        }
1296
        if (params.main_gpu.size() > 1 || params.main_gpu != cmd_params_defaults.main_gpu) {
1297
            fields.emplace_back("main_gpu");
1298
        }
1299
        if (params.split_mode.size() > 1 || params.split_mode != cmd_params_defaults.split_mode) {
1300
            fields.emplace_back("split_mode");
1301
        }
1302
        if (params.no_kv_offload.size() > 1 || params.no_kv_offload != cmd_params_defaults.no_kv_offload) {
1303
            fields.emplace_back("no_kv_offload");
1304
        }
1305
        if (params.flash_attn.size() > 1 || params.flash_attn != cmd_params_defaults.flash_attn) {
1306
            fields.emplace_back("flash_attn");
1307
        }
1308
        if (params.tensor_split.size() > 1 || params.tensor_split != cmd_params_defaults.tensor_split) {
1309
            fields.emplace_back("tensor_split");
1310
        }
1311
        if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) {
1312
            fields.emplace_back("use_mmap");
1313
        }
1314
        if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
1315
            fields.emplace_back("embeddings");
1316
        }
1317
        fields.emplace_back("test");
1318
        fields.emplace_back("t/s");
1319

1320
        fprintf(fout, "|");
1321
        for (const auto & field : fields) {
1322
            fprintf(fout, " %*s |", get_field_width(field), get_field_display_name(field).c_str());
1323
        }
1324
        fprintf(fout, "\n");
1325
        fprintf(fout, "|");
1326
        for (const auto & field : fields) {
1327
            int width = get_field_width(field);
1328
            fprintf(fout, " %s%s |", std::string(std::abs(width) - 1, '-').c_str(), width > 0 ? ":" : "-");
1329
        }
1330
        fprintf(fout, "\n");
1331
    }
1332

1333
    void print_test(const test & t) override {
1334
        std::map<std::string, std::string> vmap = t.get_map();
1335

1336
        fprintf(fout, "|");
1337
        for (const auto & field : fields) {
1338
            std::string value;
1339
            char buf[128];
1340
            if (field == "model") {
1341
                value = t.model_type;
1342
            } else if (field == "size") {
1343
                if (t.model_size < 1024*1024*1024) {
1344
                    snprintf(buf, sizeof(buf), "%.2f MiB", t.model_size / 1024.0 / 1024.0);
1345
                } else {
1346
                    snprintf(buf, sizeof(buf), "%.2f GiB", t.model_size / 1024.0 / 1024.0 / 1024.0);
1347
                }
1348
                value = buf;
1349
            } else if (field == "params") {
1350
                if (t.model_n_params < 1000*1000*1000) {
1351
                    snprintf(buf, sizeof(buf), "%.2f M", t.model_n_params / 1e6);
1352
                } else {
1353
                    snprintf(buf, sizeof(buf), "%.2f B", t.model_n_params / 1e9);
1354
                }
1355
                value = buf;
1356
            } else if (field == "backend") {
1357
                value = test::get_backend();
1358
                if (t.has_rpc) {
1359
                    value += "+RPC";
1360
                }
1361
            } else if (field == "test") {
1362
                if (t.n_prompt > 0 && t.n_gen == 0) {
1363
                    snprintf(buf, sizeof(buf), "pp%d", t.n_prompt);
1364
                } else if (t.n_gen > 0 && t.n_prompt == 0) {
1365
                    snprintf(buf, sizeof(buf), "tg%d", t.n_gen);
1366
                } else {
1367
                    snprintf(buf, sizeof(buf), "pp%d+tg%d", t.n_prompt, t.n_gen);
1368
                }
1369
                value = buf;
1370
            } else if (field == "t/s") {
1371
                snprintf(buf, sizeof(buf), "%.2f ± %.2f", t.avg_ts(), t.stdev_ts());
1372
                value = buf;
1373
            } else if (vmap.find(field) != vmap.end()) {
1374
                value = vmap.at(field);
1375
            } else {
1376
                assert(false);
1377
                exit(1);
1378
            }
1379

1380
            int width = get_field_width(field);
1381
            if (field == "t/s") {
1382
                // HACK: the utf-8 character is 2 bytes
1383
                width += 1;
1384
            }
1385
            fprintf(fout, " %*s |", width, value.c_str());
1386
        }
1387
        fprintf(fout, "\n");
1388
    }
1389

1390
    void print_footer() override {
1391
        fprintf(fout, "\nbuild: %s (%d)\n", test::build_commit.c_str(), test::build_number);
1392
    }
1393
};
1394

1395
struct sql_printer : public printer {
1396
    static std::string get_sql_field_type(const std::string & field) {
1397
        switch (test::get_field_type(field)) {
1398
            case test::STRING:
1399
                return "TEXT";
1400
            case test::BOOL:
1401
            case test::INT:
1402
                return "INTEGER";
1403
            case test::FLOAT:
1404
                return "REAL";
1405
            default:
1406
                assert(false);
1407
                exit(1);
1408
        }
1409
    }
1410

1411
    void print_header(const cmd_params & params) override {
1412
        std::vector<std::string> fields = test::get_fields();
1413
        fprintf(fout, "CREATE TABLE IF NOT EXISTS test (\n");
1414
        for (size_t i = 0; i < fields.size(); i++) {
1415
            fprintf(fout, "  %s %s%s\n", fields.at(i).c_str(), get_sql_field_type(fields.at(i)).c_str(),  i < fields.size() - 1 ? "," : "");
1416
        }
1417
        fprintf(fout, ");\n");
1418
        fprintf(fout, "\n");
1419
        (void) params;
1420
    }
1421

1422
    void print_test(const test & t) override {
1423
        fprintf(fout, "INSERT INTO test (%s) ", join(test::get_fields(), ", ").c_str());
1424
        fprintf(fout, "VALUES (");
1425
        std::vector<std::string> values = t.get_values();
1426
        for (size_t i = 0; i < values.size(); i++) {
1427
            fprintf(fout, "'%s'%s", values.at(i).c_str(), i < values.size() - 1 ? ", " : "");
1428
        }
1429
        fprintf(fout, ");\n");
1430
    }
1431
};
1432

1433
static void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) {
1434
    llama_set_n_threads(ctx, n_threads, n_threads);
1435

1436
    const llama_model * model = llama_get_model(ctx);
1437
    const int32_t n_vocab = llama_n_vocab(model);
1438

1439
    std::vector<llama_token> tokens(n_batch);
1440

1441
    int n_processed = 0;
1442

1443
    while (n_processed < n_prompt) {
1444
        int n_tokens = std::min(n_prompt - n_processed, n_batch);
1445
        tokens[0] = n_processed == 0 && llama_add_bos_token(model) ? llama_token_bos(model) : std::rand() % n_vocab;
1446
        for (int i = 1; i < n_tokens; i++) {
1447
            tokens[i] = std::rand() % n_vocab;
1448
        }
1449
        llama_decode(ctx, llama_batch_get_one(tokens.data(), n_tokens, n_past + n_processed, 0));
1450
        n_processed += n_tokens;
1451
    }
1452

1453
    llama_synchronize(ctx);
1454
}
1455

1456
static void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) {
1457
    llama_set_n_threads(ctx, n_threads, n_threads);
1458

1459
    const llama_model * model = llama_get_model(ctx);
1460
    const int32_t n_vocab = llama_n_vocab(model);
1461

1462
    llama_token token = llama_add_bos_token(model) ? llama_token_bos(model) : std::rand() % n_vocab;
1463

1464
    for (int i = 0; i < n_gen; i++) {
1465
        llama_decode(ctx, llama_batch_get_one(&token, 1, n_past + i, 0));
1466
        llama_synchronize(ctx);
1467
        token = std::rand() % n_vocab;
1468
    }
1469
}
1470

1471
static void llama_null_log_callback(enum ggml_log_level level, const char * text, void * user_data) {
1472
    (void) level;
1473
    (void) text;
1474
    (void) user_data;
1475
}
1476

1477
static std::unique_ptr<printer> create_printer(output_formats format) {
1478
    switch (format) {
1479
        case NONE:
1480
            return nullptr;
1481
        case CSV:
1482
            return std::unique_ptr<printer>(new csv_printer());
1483
        case JSON:
1484
            return std::unique_ptr<printer>(new json_printer());
1485
        case JSONL:
1486
            return std::unique_ptr<printer>(new jsonl_printer());
1487
        case MARKDOWN:
1488
            return std::unique_ptr<printer>(new markdown_printer());
1489
        case SQL:
1490
            return std::unique_ptr<printer>(new sql_printer());
1491
    }
1492
    GGML_ABORT("fatal error");
1493
}
1494

1495
int main(int argc, char ** argv) {
1496
    // try to set locale for unicode characters in markdown
1497
    setlocale(LC_CTYPE, ".UTF-8");
1498

1499
#if !defined(NDEBUG)
1500
    fprintf(stderr, "warning: asserts enabled, performance may be affected\n");
1501
#endif
1502

1503
#if (defined(_MSC_VER) && defined(_DEBUG)) || (!defined(_MSC_VER) && !defined(__OPTIMIZE__))
1504
    fprintf(stderr, "warning: debug build, performance may be affected\n");
1505
#endif
1506

1507
#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
1508
    fprintf(stderr, "warning: sanitizer enabled, performance may be affected\n");
1509
#endif
1510

1511
    cmd_params params = parse_cmd_params(argc, argv);
1512

1513
    // initialize llama.cpp
1514
    if (!params.verbose) {
1515
        llama_log_set(llama_null_log_callback, NULL);
1516
    }
1517
    llama_backend_init();
1518
    llama_numa_init(params.numa);
1519

1520
    set_process_priority(params.prio);
1521

1522
    // initialize printer
1523
    std::unique_ptr<printer> p = create_printer(params.output_format);
1524
    std::unique_ptr<printer> p_err = create_printer(params.output_format_stderr);
1525

1526
    if (p) {
1527
        p->fout = stdout;
1528
        p->print_header(params);
1529
    }
1530

1531
    if (p_err) {
1532
        p_err->fout = stderr;
1533
        p_err->print_header(params);
1534
    }
1535

1536
    std::vector<cmd_params_instance> params_instances = get_cmd_params_instances(params);
1537

1538
    llama_model * lmodel = nullptr;
1539
    const cmd_params_instance * prev_inst = nullptr;
1540

1541
    int params_idx = 0;
1542
    auto params_count = params_instances.size();
1543
    for (const auto & inst : params_instances) {
1544
        params_idx ++;
1545
        if (params.progress) {
1546
            fprintf(stderr, "llama-bench: benchmark %d/%ld: starting\n", params_idx, params_count);
1547
        }
1548
        // keep the same model between tests when possible
1549
        if (!lmodel || !prev_inst || !inst.equal_mparams(*prev_inst)) {
1550
            if (lmodel) {
1551
                llama_free_model(lmodel);
1552
            }
1553

1554
            lmodel = llama_load_model_from_file(inst.model.c_str(), inst.to_llama_mparams());
1555
            if (lmodel == NULL) {
1556
                fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, inst.model.c_str());
1557
                return 1;
1558
            }
1559
            prev_inst = &inst;
1560
        }
1561

1562
        llama_context * ctx = llama_new_context_with_model(lmodel, inst.to_llama_cparams());
1563
        if (ctx == NULL) {
1564
            fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, inst.model.c_str());
1565
            llama_free_model(lmodel);
1566
            return 1;
1567
        }
1568

1569
        test t(inst, lmodel, ctx);
1570

1571
        llama_kv_cache_clear(ctx);
1572

1573
        // cool off before the test
1574
        if (params.delay) {
1575
            std::this_thread::sleep_for(std::chrono::seconds(params.delay));
1576
        }
1577

1578
        struct ggml_threadpool_params tpp = ggml_threadpool_params_default(t.n_threads);
1579
        if (!parse_cpu_mask(t.cpu_mask, tpp.cpumask)) {
1580
            fprintf(stderr, "%s: failed to parse cpu-mask: %s\n", __func__, t.cpu_mask.c_str());
1581
            exit(1);
1582
        }
1583
        tpp.strict_cpu = t.cpu_strict;
1584
        tpp.poll       = t.poll;
1585
        tpp.prio       = params.prio;
1586

1587
        struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
1588
        if (!threadpool) {
1589
            fprintf(stderr, "%s: threadpool create failed : n_threads %d\n", __func__, tpp.n_threads);
1590
            exit(1);
1591
        }
1592

1593
        llama_attach_threadpool(ctx, threadpool, NULL);
1594

1595
        // warmup run
1596
        if (t.n_prompt > 0) {
1597
            if (params.progress) {
1598
                fprintf(stderr, "llama-bench: benchmark %d/%ld: warmup prompt run\n", params_idx, params_count);
1599
            }
1600
            //test_prompt(ctx, std::min(t.n_batch, std::min(t.n_prompt, 32)), 0, t.n_batch, t.n_threads);
1601
            test_prompt(ctx, t.n_prompt, 0, t.n_batch, t.n_threads);
1602
        }
1603
        if (t.n_gen > 0) {
1604
            if (params.progress) {
1605
                fprintf(stderr, "llama-bench: benchmark %d/%ld: warmup generation run\n", params_idx, params_count);
1606
            }
1607
            test_gen(ctx, 1, 0, t.n_threads);
1608
        }
1609

1610
        for (int i = 0; i < params.reps; i++) {
1611
            llama_kv_cache_clear(ctx);
1612

1613
            uint64_t t_start = get_time_ns();
1614

1615
            if (t.n_prompt > 0) {
1616
                if (params.progress) {
1617
                    fprintf(stderr, "llama-bench: benchmark %d/%ld: prompt run %d/%d\n", params_idx, params_count, i + 1, params.reps);
1618
                }
1619
                test_prompt(ctx, t.n_prompt, 0, t.n_batch, t.n_threads);
1620
            }
1621
            if (t.n_gen > 0) {
1622
                if (params.progress) {
1623
                    fprintf(stderr, "llama-bench: benchmark %d/%ld: generation run %d/%d\n", params_idx, params_count, i + 1, params.reps);
1624
                }
1625
                test_gen(ctx, t.n_gen, t.n_prompt, t.n_threads);
1626
            }
1627

1628
            uint64_t t_ns = get_time_ns() - t_start;
1629
            t.samples_ns.push_back(t_ns);
1630
        }
1631

1632
        if (p) {
1633
            p->print_test(t);
1634
            fflush(p->fout);
1635
        }
1636

1637
        if (p_err) {
1638
            p_err->print_test(t);
1639
            fflush(p_err->fout);
1640
        }
1641

1642
        llama_perf_context_print(ctx);
1643

1644
        llama_free(ctx);
1645

1646
        ggml_threadpool_free(threadpool);
1647
    }
1648

1649
    llama_free_model(lmodel);
1650

1651
    if (p) {
1652
        p->print_footer();
1653
    }
1654

1655
    if (p_err) {
1656
        p_err->print_footer();
1657
    }
1658

1659
    llama_backend_free();
1660

1661
    return 0;
1662
}
1663