llmware

1

2
"""This example demonstrates several leading open source chat models running in 4-bit GGUF on local laptop."""
3

4
import time
5
import re
6
from llmware.prompts import Prompt
7

8

9
# Run the benchmark test
10
def run_test(model_name, prompt_list):
11

12
    print(f"\n > Loading model '{model_name}'")
13

14
    prompter = Prompt().load_model(model_name)
15

16
    for i, entry in enumerate(prompt_list):
17

18
        start_time = time.time()
19
        print("\n")
20
        print(f"query - {i+1} - {entry['query']}")
21

22
        response = prompter.prompt_main(entry["query"])
23

24
        # Print results
25
        time_taken = round(time.time() - start_time, 2)
26
        llm_response = re.sub("[\n\n]", "\n", response['llm_response'])
27
        print(f"llm_response - {i+1} - {llm_response}")
28
        print(f"time_taken - {i+1} - {time_taken}")
29

30
    return 0
31

32

33
if __name__ == "__main__":
34

35
    #   Example open-ended queries with no context - looking for chat model to draw on general know-how and ability
36
    #   to look at a problem conceptually, with focus on language understanding without specific focus on facts
37

38
    ds = [
39
        {"query": "I am interested in gaining an understanding of the banking industry. What topics should I research?",
40
         "context": "", "answer": "NO_GOLD_ANSWER"},
41
        {"query": "What are some tips for creating a successful business plan?", "context": "", "answer": ""},
42
        {"query": "What do you think about the recent news about Microsoft\u2019s GitHub acquisition?", "context": "",
43
         "answer": ""},
44
        {"query": "What are the best books to read for a class on American literature?", "context": "", "answer": ""},
45
        {"query" : "What is the most important thing I should know about my local school district?", "context": "", "answer": ""},
46
        {"query": "Can you recommend some good books for me to read?", "context": "", "answer": ""},
47
        {"query": "What are the differences between the four principal cloud computing service models: IaaS, PaaS, SaaS, and CaaS?",
48
         "context": "", "answer": ""},
49
        {"query": "I've heard a lot of good things about the iPhone. Should I buy one?", "context": "", "answer": ""},
50
        {"query": "What is the difference between a sociological and psychological approach to social problems?",
51
         "context": "", "answer": ""},
52
        {"query": "What job opportunities are available for someone with degree in chemistry?", "context": "", "answer": ""},
53
        {"query": "I'd like to know how to get the most out of my money in the stock market?", "context": "", "answer": ""},
54
        {"query": "How do I know if my computer is secure?", "context": "", "answer": ""},
55
        {"query": "I'm writing an essay on the importance of reading. What are some good questions to ask "
56
                  "in this type of essay?", "context": "", "answer": ""},
57
        {"query": "What is the best way for small businesses to raise capital for their operations?", "context": "", "answer": ""},
58
        {"query": "I want to start a blog but don't know what to write about. What should I do?", "context": "", "answer": ""},
59
        {"query": "What are the best ways to learn a new language?", "context": "", "answer": ""},
60
        {"query": "I'm having some problems with my computer. What are some of the most common computer problems"
61
                  " and how can I fix them?", "context": "", "answer": ""},
62
        {"query": "How can I improve my credit score?", "context": "", "answer": ""},
63
        {"query": "How can I build confidence in my ability to write articles and be a good writer?",
64
         "context": "", "answer": ""},
65
        {"query": "How do I negotiate a salary raise?", "context": "", "answer": ""},
66
        {"query": "What's the best way to learn how to write a good essay?", "context": "", "answer": ""},
67
        {"query": "I'm a little nervous about taking my first trip abroad. What do I need to know?", "context": "",
68
         "answer": ""},
69
        {"query": "What is the difference between a dividend and a capital gain?", "context": "", "answer": ""},
70
        {"query": "What are the best ways to make a house a home?", "context": "", "answer": ""},
71
        {"query": "I have a question about ecology. What is the difference between a population and a community?",
72
         "context": "", "answer": ""}
73
        ]
74

75
    #   please note that these models will produce multiple bulletpoints and paragraph length answers, so
76
    #   it can take ~30 seconds for each response on a typical Mac M1 laptop
77

78
    #   we have tested four leading open source 7B chat models
79
    #   for full citations and links, please go to www.huggingface.co/llmware/bonchon model repository
80
    #   If you don't know TheBloke ... now you do!   He has a large library of GGUF models ...
81
    #       -- TheBloke/OpenHermes-2.5-Mistral-7B-GGUF
82
    #       -- TheBloke/zephyr-7B-beta-GGUF
83
    #       -- TheBloke/Starling-LM-7B-alpha-GGUF
84
    #       --  TheBloke/Llama-2-7B-Chat-GGUF
85

86
    output = run_test("TheBloke/Llama-2-7B-Chat-GGUF",ds)
87