llama-index

1
"""
2
Utility Tools for the Portkey Class.
3

4
This file module contains a collection of utility functions designed to enhance
5
the functionality and usability of the Portkey class
6
"""
7

8
from typing import TYPE_CHECKING, List
9

10
from llama_index.legacy.core.llms.types import LLMMetadata
11
from llama_index.legacy.llms.anthropic import Anthropic
12
from llama_index.legacy.llms.anthropic_utils import CLAUDE_MODELS
13
from llama_index.legacy.llms.openai import OpenAI
14
from llama_index.legacy.llms.openai_utils import (
15
    AZURE_TURBO_MODELS,
16
    GPT3_5_MODELS,
17
    GPT3_MODELS,
18
    GPT4_MODELS,
19
    TURBO_MODELS,
20
)
21

22
if TYPE_CHECKING:
23
    from portkey import (
24
        LLMOptions,
25
        PortkeyResponse,
26
    )
27

28

29
IMPORT_ERROR_MESSAGE = (
30
    "Portkey is not installed.Please install it with `pip install portkey-ai`."
31
)
32

33

34
DISCONTINUED_MODELS = {
35
    "code-davinci-002": 8001,
36
    "code-davinci-001": 8001,
37
    "code-cushman-002": 2048,
38
    "code-cushman-001": 2048,
39
}
40

41
DEFAULT_MODEL = "gpt-3.5-turbo"
42

43
AVAILABLE_INTEGRATIONS = (OpenAI, Anthropic)
44

45
CLUADE_MODEL_FULLVERSION_MAP = {
46
    "claude-instant-1": "claude-instant-1.2",
47
    "claude-2": "claude-2.0",
48
}
49

50
ALL_AVAILABLE_MODELS = {
51
    **GPT4_MODELS,
52
    **TURBO_MODELS,
53
    **GPT3_5_MODELS,
54
    **GPT3_MODELS,
55
    **AZURE_TURBO_MODELS,
56
    **CLAUDE_MODELS,
57
}
58

59
CHAT_MODELS = {
60
    **GPT4_MODELS,
61
    **TURBO_MODELS,
62
    **AZURE_TURBO_MODELS,
63
}
64

65

66
def is_chat_model(model: str) -> bool:
67
    """Check if a given model is a chat-based language model.
68

69
    This function takes a model name or identifier as input and determines whether
70
    the model is designed for chat-based language generation, conversation, or
71
    interaction.
72

73
    Args:
74
        model (str): The name or identifier of the model to be checked.
75

76
    Returns:
77
        bool: True if the provided model is a chat-based language model,
78
        False otherwise.
79
    """
80
    return model in CHAT_MODELS
81

82

83
def modelname_to_contextsize(modelname: str) -> int:
84
    """Calculate the maximum number of tokens possible to generate for a model.
85

86
    Args:
87
        modelname: The modelname we want to know the context size for.
88

89
    Returns:
90
        The maximum context size
91

92
    Example:
93
        .. code-block:: python
94

95
            max_tokens = modelname_to_contextsize("text-davinci-003")
96
    """
97
    # handling finetuned models
98
    if "ft-" in modelname:  # legacy fine-tuning
99
        modelname = modelname.split(":")[0]
100
    elif modelname.startswith("ft:"):
101
        modelname = modelname.split(":")[1]
102

103
    if modelname in DISCONTINUED_MODELS:
104
        raise ValueError(
105
            f"Model {modelname} has been discontinued. " "Please choose another model."
106
        )
107

108
    context_size = ALL_AVAILABLE_MODELS.get(modelname, None)
109

110
    if context_size is None:
111
        raise ValueError(
112
            f"Unknown model: {modelname}. Please provide a valid model name."
113
            "Known models are: " + ", ".join(ALL_AVAILABLE_MODELS.keys())
114
        )
115

116
    return context_size
117

118

119
def generate_llm_metadata(llm: "LLMOptions") -> LLMMetadata:
120
    """
121
    Generate metadata for a Language Model (LLM) instance.
122

123
    This function takes an instance of a Language Model (LLM) and generates
124
    metadata based on the provided instance. The metadata includes information
125
    such as the context window, number of output tokens, chat model status,
126
    and model name.
127

128
    Parameters:
129
        llm (LLM): An instance of a Language Model (LLM) from which metadata
130
            will be generated.
131

132
    Returns:
133
        LLMMetadata: A data structure containing metadata attributes such as
134
            context window, number of output tokens, chat model status, and
135
            model name.
136

137
    Raises:
138
        ValueError: If the provided 'llm' is not an instance of
139
        llama_index.llms.base.LLM.
140
    """
141
    try:
142
        from portkey import LLMOptions
143
    except ImportError as exc:
144
        raise ImportError(IMPORT_ERROR_MESSAGE) from exc
145
    if not isinstance(llm, LLMOptions):
146
        raise ValueError("llm must be an instance of portkey.LLMOptions")
147

148
    return LLMMetadata(
149
        _context_window=modelname_to_contextsize(llm.model or ""),
150
        is_chat_model=is_chat_model(llm.model or ""),
151
        model_name=llm.model,
152
    )
153

154

155
def get_llm(response: "PortkeyResponse", llms: List["LLMOptions"]) -> "LLMOptions":
156
    # TODO: Update this logic over here.
157
    try:
158
        from portkey import LLMOptions
159
    except ImportError as exc:
160
        raise ImportError(IMPORT_ERROR_MESSAGE) from exc
161
    fallback_llm = LLMOptions.construct()
162
    for llm in llms:
163
        model = llm.model
164

165
        if model == response.model:
166
            fallback_llm = llm
167
            break
168

169
    if fallback_llm is None:
170
        raise ValueError("Failed to get the fallback LLM")
171
    return fallback_llm
172