promptfoo

1
export const DEFAULT_PROMPTS = `Your first prompt goes here
2
---
3
Next prompt goes here. You can substitute variables like this: {{var1}} {{var2}} {{var3}}
4
---
5
This is the next prompt.
6

7
These prompts are nunjucks templates, so you can use logic like this:
8
{% if var1 %}
9
  {{ var1 }}
10
{% endif %}
11
---
12
[
13
  {"role": "system", "content": "This is another prompt. JSON is supported."},
14
  {"role": "user", "content": "Using this format, you may construct multi-shot OpenAI prompts"}
15
  {"role": "user", "content": "Variable substitution still works: {{ var3 }}"}
16
]
17
---
18
If you prefer, you can break prompts into multiple files (make sure to edit promptfooconfig.yaml accordingly)
19
`;
20

21
export const DEFAULT_YAML_CONFIG = `# This configuration compares LLM output of 2 prompts x 2 GPT models across 3 test cases.
22
# Learn more: https://promptfoo.dev/docs/configuration/guide
23
description: 'My first eval'
24

25
prompts:
26
  - "Write a tweet about {{topic}}"
27
  - "Write a very concise, funny tweet about {{topic}}"
28

29
providers:
30
  - openai:gpt-3.5-turbo-0613
31
  - openai:gpt-4
32

33
tests:
34
  - vars:
35
      topic: bananas
36

37
  - vars:
38
      topic: avocado toast
39
    assert:
40
      # For more information on assertions, see https://promptfoo.dev/docs/configuration/expected-outputs
41
      - type: icontains
42
        value: avocado
43
      - type: javascript
44
        value: 1 / (output.length + 1)  # prefer shorter outputs
45

46
  - vars:
47
      topic: new york city
48
    assert:
49
      # For more information on model-graded evals, see https://promptfoo.dev/docs/configuration/expected-outputs/model-graded
50
      - type: llm-rubric
51
        value: ensure that the output is funny
52
`;
53

54
export const DEFAULT_README = `To get started, set your OPENAI_API_KEY environment variable.
55

56
Next, edit promptfooconfig.yaml.
57

58
Then run:
59
\`\`\`
60
promptfoo eval
61
\`\`\`
62

63
Afterwards, you can view the results by running \`promptfoo view\`
64
`;
65