llama

1
#!/usr/bin/env python3
2

3
import logging
4
import argparse
5
import os
6
import subprocess
7
import sys
8

9
import yaml
10

11
logger = logging.getLogger("run-with-preset")
12

13
CLI_ARGS_LLAMA_CLI_PERPLEXITY = [
14
    "batch-size", "cfg-negative-prompt", "cfg-scale", "chunks", "color", "ctx-size", "escape",
15
    "export", "file", "frequency-penalty", "grammar", "grammar-file", "hellaswag",
16
    "hellaswag-tasks", "ignore-eos", "in-prefix", "in-prefix-bos", "in-suffix",
17
    "interactive", "interactive-first", "keep", "logdir", "logit-bias", "lora", "lora-base",
18
    "low-vram", "main-gpu", "memory-f32", "mirostat", "mirostat-ent", "mirostat-lr", "mlock",
19
    "model", "multiline-input", "n-gpu-layers", "n-predict", "no-mmap", "no-mul-mat-q",
20
    "np-penalize-nl", "numa", "ppl-output-type", "ppl-stride", "presence-penalty", "prompt",
21
    "prompt-cache", "prompt-cache-all", "prompt-cache-ro", "repeat-last-n",
22
    "repeat-penalty", "reverse-prompt", "rope-freq-base", "rope-freq-scale", "rope-scale", "seed",
23
    "simple-io", "tensor-split", "threads", "temp", "tfs", "top-k", "top-p", "typical",
24
    "verbose-prompt"
25
]
26

27
CLI_ARGS_LLAMA_BENCH = [
28
    "batch-size", "memory-f32", "low-vram", "model", "mul-mat-q", "n-gen", "n-gpu-layers",
29
    "n-prompt", "output", "repetitions", "tensor-split", "threads", "verbose"
30
]
31

32
CLI_ARGS_LLAMA_SERVER = [
33
    "alias", "batch-size", "ctx-size", "embedding", "host", "memory-f32", "lora", "lora-base",
34
    "low-vram", "main-gpu", "mlock", "model", "n-gpu-layers", "n-probs", "no-mmap", "no-mul-mat-q",
35
    "numa", "path", "port", "rope-freq-base", "timeout", "rope-freq-scale", "tensor-split",
36
    "threads", "verbose"
37
]
38

39
description = """Run llama.cpp binaries with presets from YAML file(s).
40
To specify which binary should be run, specify the "binary" property (llama-cli, llama-perplexity, llama-bench, and llama-server are supported).
41
To get a preset file template, run a llama.cpp binary with the "--logdir" CLI argument.
42

43
Formatting considerations:
44
- The YAML property names are the same as the CLI argument names of the corresponding binary.
45
- Properties must use the long name of their corresponding llama.cpp CLI arguments.
46
- Like the llama.cpp binaries the property names do not differentiate between hyphens and underscores.
47
- Flags must be defined as "<PROPERTY_NAME>: true" to be effective.
48
- To define the logit_bias property, the expected format is "<TOKEN_ID>: <BIAS>" in the "logit_bias" namespace.
49
- To define multiple "reverse_prompt" properties simultaneously the expected format is a list of strings.
50
- To define a tensor split, pass a list of floats.
51
"""
52
usage = "run-with-preset.py [-h] [yaml_files ...] [--<ARG_NAME> <ARG_VALUE> ...]"
53
epilog = ("  --<ARG_NAME> specify additional CLI ars to be passed to the binary (override all preset files). "
54
          "Unknown args will be ignored.")
55

56
parser = argparse.ArgumentParser(
57
    description=description, usage=usage, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
58
parser.add_argument("-bin", "--binary", help="The binary to run.")
59
parser.add_argument("yaml_files", nargs="*",
60
                    help="Arbitrary number of YAML files from which to read preset values. "
61
                    "If two files specify the same values the later one will be used.")
62
parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
63

64
known_args, unknown_args = parser.parse_known_args()
65

66
if not known_args.yaml_files and not unknown_args:
67
    parser.print_help()
68
    sys.exit(0)
69

70
logging.basicConfig(level=logging.DEBUG if known_args.verbose else logging.INFO)
71

72
props = dict()
73

74
for yaml_file in known_args.yaml_files:
75
    with open(yaml_file, "r") as f:
76
        props.update(yaml.load(f, yaml.SafeLoader))
77

78
props = {prop.replace("_", "-"): val for prop, val in props.items()}
79

80
binary = props.pop("binary", "llama-cli")
81
if known_args.binary:
82
    binary = known_args.binary
83

84
if os.path.exists(f"./{binary}"):
85
    binary = f"./{binary}"
86

87
if binary.lower().endswith("llama-cli") or binary.lower().endswith("llama-perplexity"):
88
    cli_args = CLI_ARGS_LLAMA_CLI_PERPLEXITY
89
elif binary.lower().endswith("llama-bench"):
90
    cli_args = CLI_ARGS_LLAMA_BENCH
91
elif binary.lower().endswith("llama-server"):
92
    cli_args = CLI_ARGS_LLAMA_SERVER
93
else:
94
    logger.error(f"Unknown binary: {binary}")
95
    sys.exit(1)
96

97
command_list = [binary]
98

99
for cli_arg in cli_args:
100
    value = props.pop(cli_arg, None)
101

102
    if not value or value == -1:
103
        continue
104

105
    if cli_arg == "logit-bias":
106
        for token, bias in value.items():
107
            command_list.append("--logit-bias")
108
            command_list.append(f"{token}{bias:+}")
109
        continue
110

111
    if cli_arg == "reverse-prompt" and not isinstance(value, str):
112
        for rp in value:
113
            command_list.append("--reverse-prompt")
114
            command_list.append(str(rp))
115
        continue
116

117
    command_list.append(f"--{cli_arg}")
118

119
    if cli_arg == "tensor-split":
120
        command_list.append(",".join([str(v) for v in value]))
121
        continue
122

123
    value = str(value)
124

125
    if value != "True":
126
        command_list.append(str(value))
127

128
num_unused = len(props)
129
if num_unused > 10:
130
    logger.info(f"The preset file contained a total of {num_unused} unused properties.")
131
elif num_unused > 0:
132
    logger.info("The preset file contained the following unused properties:")
133
    for prop, value in props.items():
134
        logger.info(f"  {prop}: {value}")
135

136
command_list += unknown_args
137

138
sp = subprocess.Popen(command_list)
139

140
while sp.returncode is None:
141
    try:
142
        sp.wait()
143
    except KeyboardInterrupt:
144
        pass
145

146
sys.exit(sp.returncode)
147