ollama

1
package api
2

3
import (
4
	"encoding/json"
5
	"fmt"
6
	"math"
7
	"os"
8
	"reflect"
9
	"strconv"
10
	"strings"
11
	"time"
12
)
13

14
type StatusError struct {
15
	StatusCode   int
16
	Status       string
17
	ErrorMessage string `json:"error"`
18
}
19

20
func (e StatusError) Error() string {
21
	switch {
22
	case e.Status != "" && e.ErrorMessage != "":
23
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
24
	case e.Status != "":
25
		return e.Status
26
	case e.ErrorMessage != "":
27
		return e.ErrorMessage
28
	default:
29
		// this should not happen
30
		return "something went wrong, please see the ollama server logs for details"
31
	}
32
}
33

34
type ImageData []byte
35

36
type GenerateRequest struct {
37
	Model     string      `json:"model"`
38
	Prompt    string      `json:"prompt"`
39
	System    string      `json:"system"`
40
	Template  string      `json:"template"`
41
	Context   []int       `json:"context,omitempty"`
42
	Stream    *bool       `json:"stream,omitempty"`
43
	Raw       bool        `json:"raw,omitempty"`
44
	Format    string      `json:"format"`
45
	KeepAlive *Duration   `json:"keep_alive,omitempty"`
46
	Images    []ImageData `json:"images,omitempty"`
47

48
	Options map[string]interface{} `json:"options"`
49
}
50

51
type ChatRequest struct {
52
	Model     string    `json:"model"`
53
	Messages  []Message `json:"messages"`
54
	Stream    *bool     `json:"stream,omitempty"`
55
	Format    string    `json:"format"`
56
	KeepAlive *Duration `json:"keep_alive,omitempty"`
57

58
	Options map[string]interface{} `json:"options"`
59
}
60

61
type Message struct {
62
	Role    string      `json:"role"` // one of ["system", "user", "assistant"]
63
	Content string      `json:"content"`
64
	Images  []ImageData `json:"images,omitempty"`
65
}
66

67
type ChatResponse struct {
68
	Model     string    `json:"model"`
69
	CreatedAt time.Time `json:"created_at"`
70
	Message   Message   `json:"message"`
71

72
	Done bool `json:"done"`
73

74
	Metrics
75
}
76

77
type Metrics struct {
78
	TotalDuration      time.Duration `json:"total_duration,omitempty"`
79
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
80
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
81
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
82
	EvalCount          int           `json:"eval_count,omitempty"`
83
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
84
}
85

86
// Options specified in GenerateRequest, if you add a new option here add it to the API docs also
87
type Options struct {
88
	Runner
89

90
	// Predict options used at runtime
91
	NumKeep          int      `json:"num_keep,omitempty"`
92
	Seed             int      `json:"seed,omitempty"`
93
	NumPredict       int      `json:"num_predict,omitempty"`
94
	TopK             int      `json:"top_k,omitempty"`
95
	TopP             float32  `json:"top_p,omitempty"`
96
	TFSZ             float32  `json:"tfs_z,omitempty"`
97
	TypicalP         float32  `json:"typical_p,omitempty"`
98
	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
99
	Temperature      float32  `json:"temperature,omitempty"`
100
	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
101
	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
102
	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
103
	Mirostat         int      `json:"mirostat,omitempty"`
104
	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
105
	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
106
	PenalizeNewline  bool     `json:"penalize_newline,omitempty"`
107
	Stop             []string `json:"stop,omitempty"`
108
}
109

110
// Runner options which must be set when the model is loaded into memory
111
type Runner struct {
112
	UseNUMA            bool    `json:"numa,omitempty"`
113
	NumCtx             int     `json:"num_ctx,omitempty"`
114
	NumBatch           int     `json:"num_batch,omitempty"`
115
	NumGQA             int     `json:"num_gqa,omitempty"`
116
	NumGPU             int     `json:"num_gpu,omitempty"`
117
	MainGPU            int     `json:"main_gpu,omitempty"`
118
	LowVRAM            bool    `json:"low_vram,omitempty"`
119
	F16KV              bool    `json:"f16_kv,omitempty"`
120
	LogitsAll          bool    `json:"logits_all,omitempty"`
121
	VocabOnly          bool    `json:"vocab_only,omitempty"`
122
	UseMMap            bool    `json:"use_mmap,omitempty"`
123
	UseMLock           bool    `json:"use_mlock,omitempty"`
124
	RopeFrequencyBase  float32 `json:"rope_frequency_base,omitempty"`
125
	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
126
	NumThread          int     `json:"num_thread,omitempty"`
127
}
128

129
type EmbeddingRequest struct {
130
	Model     string    `json:"model"`
131
	Prompt    string    `json:"prompt"`
132
	KeepAlive *Duration `json:"keep_alive,omitempty"`
133

134
	Options map[string]interface{} `json:"options"`
135
}
136

137
type EmbeddingResponse struct {
138
	Embedding []float64 `json:"embedding"`
139
}
140

141
type CreateRequest struct {
142
	Model     string `json:"model"`
143
	Path      string `json:"path"`
144
	Modelfile string `json:"modelfile"`
145
	Stream    *bool  `json:"stream,omitempty"`
146

147
	// Name is deprecated, see Model
148
	Name string `json:"name"`
149
}
150

151
type DeleteRequest struct {
152
	Model string `json:"model"`
153

154
	// Name is deprecated, see Model
155
	Name string `json:"name"`
156
}
157

158
type ShowRequest struct {
159
	Model    string `json:"model"`
160
	System   string `json:"system"`
161
	Template string `json:"template"`
162

163
	Options map[string]interface{} `json:"options"`
164

165
	// Name is deprecated, see Model
166
	Name string `json:"name"`
167
}
168

169
type ShowResponse struct {
170
	License    string       `json:"license,omitempty"`
171
	Modelfile  string       `json:"modelfile,omitempty"`
172
	Parameters string       `json:"parameters,omitempty"`
173
	Template   string       `json:"template,omitempty"`
174
	System     string       `json:"system,omitempty"`
175
	Details    ModelDetails `json:"details,omitempty"`
176
	Messages   []Message    `json:"messages,omitempty"`
177
}
178

179
type CopyRequest struct {
180
	Source      string `json:"source"`
181
	Destination string `json:"destination"`
182
}
183

184
type PullRequest struct {
185
	Model    string `json:"model"`
186
	Insecure bool   `json:"insecure,omitempty"`
187
	Username string `json:"username"`
188
	Password string `json:"password"`
189
	Stream   *bool  `json:"stream,omitempty"`
190

191
	// Name is deprecated, see Model
192
	Name string `json:"name"`
193
}
194

195
type ProgressResponse struct {
196
	Status    string `json:"status"`
197
	Digest    string `json:"digest,omitempty"`
198
	Total     int64  `json:"total,omitempty"`
199
	Completed int64  `json:"completed,omitempty"`
200
}
201

202
type PushRequest struct {
203
	Model    string `json:"model"`
204
	Insecure bool   `json:"insecure,omitempty"`
205
	Username string `json:"username"`
206
	Password string `json:"password"`
207
	Stream   *bool  `json:"stream,omitempty"`
208

209
	// Name is deprecated, see Model
210
	Name string `json:"name"`
211
}
212

213
type ListResponse struct {
214
	Models []ModelResponse `json:"models"`
215
}
216

217
type ModelResponse struct {
218
	Name       string       `json:"name"`
219
	Model      string       `json:"model"`
220
	ModifiedAt time.Time    `json:"modified_at"`
221
	Size       int64        `json:"size"`
222
	Digest     string       `json:"digest"`
223
	Details    ModelDetails `json:"details,omitempty"`
224
}
225

226
type TokenResponse struct {
227
	Token string `json:"token"`
228
}
229

230
type GenerateResponse struct {
231
	Model     string    `json:"model"`
232
	CreatedAt time.Time `json:"created_at"`
233
	Response  string    `json:"response"`
234

235
	Done    bool  `json:"done"`
236
	Context []int `json:"context,omitempty"`
237

238
	Metrics
239
}
240

241
type ModelDetails struct {
242
	ParentModel       string   `json:"parent_model"`
243
	Format            string   `json:"format"`
244
	Family            string   `json:"family"`
245
	Families          []string `json:"families"`
246
	ParameterSize     string   `json:"parameter_size"`
247
	QuantizationLevel string   `json:"quantization_level"`
248
}
249

250
func (m *Metrics) Summary() {
251
	if m.TotalDuration > 0 {
252
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)
253
	}
254

255
	if m.LoadDuration > 0 {
256
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", m.LoadDuration)
257
	}
258

259
	if m.PromptEvalCount > 0 {
260
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", m.PromptEvalCount)
261
	}
262

263
	if m.PromptEvalDuration > 0 {
264
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", m.PromptEvalDuration)
265
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(m.PromptEvalCount)/m.PromptEvalDuration.Seconds())
266
	}
267

268
	if m.EvalCount > 0 {
269
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", m.EvalCount)
270
	}
271

272
	if m.EvalDuration > 0 {
273
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", m.EvalDuration)
274
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(m.EvalCount)/m.EvalDuration.Seconds())
275
	}
276
}
277

278
var ErrInvalidOpts = fmt.Errorf("invalid options")
279

280
func (opts *Options) FromMap(m map[string]interface{}) error {
281
	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
282
	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct
283

284
	// build map of json struct tags to their types
285
	jsonOpts := make(map[string]reflect.StructField)
286
	for _, field := range reflect.VisibleFields(typeOpts) {
287
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
288
		if jsonTag != "" {
289
			jsonOpts[jsonTag] = field
290
		}
291
	}
292

293
	invalidOpts := []string{}
294
	for key, val := range m {
295
		if opt, ok := jsonOpts[key]; ok {
296
			field := valueOpts.FieldByName(opt.Name)
297
			if field.IsValid() && field.CanSet() {
298
				if val == nil {
299
					continue
300
				}
301

302
				switch field.Kind() {
303
				case reflect.Int:
304
					switch t := val.(type) {
305
					case int64:
306
						field.SetInt(t)
307
					case float64:
308
						// when JSON unmarshals numbers, it uses float64, not int
309
						field.SetInt(int64(t))
310
					default:
311
						return fmt.Errorf("option %q must be of type integer", key)
312
					}
313
				case reflect.Bool:
314
					val, ok := val.(bool)
315
					if !ok {
316
						return fmt.Errorf("option %q must be of type boolean", key)
317
					}
318
					field.SetBool(val)
319
				case reflect.Float32:
320
					// JSON unmarshals to float64
321
					val, ok := val.(float64)
322
					if !ok {
323
						return fmt.Errorf("option %q must be of type float32", key)
324
					}
325
					field.SetFloat(val)
326
				case reflect.String:
327
					val, ok := val.(string)
328
					if !ok {
329
						return fmt.Errorf("option %q must be of type string", key)
330
					}
331
					field.SetString(val)
332
				case reflect.Slice:
333
					// JSON unmarshals to []interface{}, not []string
334
					val, ok := val.([]interface{})
335
					if !ok {
336
						return fmt.Errorf("option %q must be of type array", key)
337
					}
338
					// convert []interface{} to []string
339
					slice := make([]string, len(val))
340
					for i, item := range val {
341
						str, ok := item.(string)
342
						if !ok {
343
							return fmt.Errorf("option %q must be of an array of strings", key)
344
						}
345
						slice[i] = str
346
					}
347
					field.Set(reflect.ValueOf(slice))
348
				default:
349
					return fmt.Errorf("unknown type loading config params: %v", field.Kind())
350
				}
351
			}
352
		} else {
353
			invalidOpts = append(invalidOpts, key)
354
		}
355
	}
356

357
	if len(invalidOpts) > 0 {
358
		return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", "))
359
	}
360
	return nil
361
}
362

363
func DefaultOptions() Options {
364
	return Options{
365
		// options set on request to runner
366
		NumPredict:       -1,
367
		NumKeep:          0,
368
		Temperature:      0.8,
369
		TopK:             40,
370
		TopP:             0.9,
371
		TFSZ:             1.0,
372
		TypicalP:         1.0,
373
		RepeatLastN:      64,
374
		RepeatPenalty:    1.1,
375
		PresencePenalty:  0.0,
376
		FrequencyPenalty: 0.0,
377
		Mirostat:         0,
378
		MirostatTau:      5.0,
379
		MirostatEta:      0.1,
380
		PenalizeNewline:  true,
381
		Seed:             -1,
382

383
		Runner: Runner{
384
			// options set when the model is loaded
385
			NumCtx:             2048,
386
			RopeFrequencyBase:  10000.0,
387
			RopeFrequencyScale: 1.0,
388
			NumBatch:           512,
389
			NumGPU:             -1, // -1 here indicates that NumGPU should be set dynamically
390
			NumGQA:             1,
391
			NumThread:          0, // let the runtime decide
392
			LowVRAM:            false,
393
			F16KV:              true,
394
			UseMLock:           false,
395
			UseMMap:            true,
396
			UseNUMA:            false,
397
		},
398
	}
399
}
400

401
type Duration struct {
402
	time.Duration
403
}
404

405
func (d *Duration) UnmarshalJSON(b []byte) (err error) {
406
	var v any
407
	if err := json.Unmarshal(b, &v); err != nil {
408
		return err
409
	}
410

411
	d.Duration = 5 * time.Minute
412

413
	switch t := v.(type) {
414
	case float64:
415
		if t < 0 {
416
			d.Duration = time.Duration(math.MaxInt64)
417
		} else {
418
			d.Duration = time.Duration(t * float64(time.Second))
419
		}
420
	case string:
421
		d.Duration, err = time.ParseDuration(t)
422
		if err != nil {
423
			return err
424
		}
425
		if d.Duration < 0 {
426
			d.Duration = time.Duration(math.MaxInt64)
427
		}
428
	}
429

430
	return nil
431
}
432

433
// FormatParams converts specified parameter options to their correct types
434
func FormatParams(params map[string][]string) (map[string]interface{}, error) {
435
	opts := Options{}
436
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
437
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct
438

439
	// build map of json struct tags to their types
440
	jsonOpts := make(map[string]reflect.StructField)
441
	for _, field := range reflect.VisibleFields(typeOpts) {
442
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
443
		if jsonTag != "" {
444
			jsonOpts[jsonTag] = field
445
		}
446
	}
447

448
	out := make(map[string]interface{})
449
	// iterate params and set values based on json struct tags
450
	for key, vals := range params {
451
		if opt, ok := jsonOpts[key]; !ok {
452
			return nil, fmt.Errorf("unknown parameter '%s'", key)
453
		} else {
454
			field := valueOpts.FieldByName(opt.Name)
455
			if field.IsValid() && field.CanSet() {
456
				switch field.Kind() {
457
				case reflect.Float32:
458
					floatVal, err := strconv.ParseFloat(vals[0], 32)
459
					if err != nil {
460
						return nil, fmt.Errorf("invalid float value %s", vals)
461
					}
462

463
					out[key] = float32(floatVal)
464
				case reflect.Int:
465
					intVal, err := strconv.ParseInt(vals[0], 10, 64)
466
					if err != nil {
467
						return nil, fmt.Errorf("invalid int value %s", vals)
468
					}
469

470
					out[key] = intVal
471
				case reflect.Bool:
472
					boolVal, err := strconv.ParseBool(vals[0])
473
					if err != nil {
474
						return nil, fmt.Errorf("invalid bool value %s", vals)
475
					}
476

477
					out[key] = boolVal
478
				case reflect.String:
479
					out[key] = vals[0]
480
				case reflect.Slice:
481
					// TODO: only string slices are supported right now
482
					out[key] = vals
483
				default:
484
					return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
485
				}
486
			}
487
		}
488
	}
489

490
	return out, nil
491
}
492