llama-index

1
from typing import Any, Callable, Dict, Optional, Sequence
2

3
from llama_index.legacy.bridge.pydantic import Field, PrivateAttr
4
from llama_index.legacy.callbacks import CallbackManager
5
from llama_index.legacy.core.llms.types import (
6
    ChatMessage,
7
    ChatResponse,
8
    ChatResponseAsyncGen,
9
    ChatResponseGen,
10
    CompletionResponse,
11
    CompletionResponseAsyncGen,
12
    CompletionResponseGen,
13
    LLMMetadata,
14
)
15
from llama_index.legacy.llms.base import (
16
    llm_chat_callback,
17
    llm_completion_callback,
18
)
19
from llama_index.legacy.llms.generic_utils import (
20
    completion_response_to_chat_response,
21
    stream_completion_response_to_chat_response,
22
)
23
from llama_index.legacy.llms.llama_utils import completion_to_prompt, messages_to_prompt
24
from llama_index.legacy.llms.llm import LLM
25
from llama_index.legacy.llms.sagemaker_llm_endpoint_utils import (
26
    BaseIOHandler,
27
    IOHandler,
28
)
29
from llama_index.legacy.types import BaseOutputParser, PydanticProgramMode
30
from llama_index.legacy.utilities.aws_utils import get_aws_service_client
31

32
DEFAULT_IO_HANDLER = IOHandler()
33
LLAMA_MESSAGES_TO_PROMPT = messages_to_prompt
34
LLAMA_COMPLETION_TO_PROMPT = completion_to_prompt
35

36

37
class SageMakerLLM(LLM):
38
    endpoint_name: str = Field(description="SageMaker LLM endpoint name")
39
    endpoint_kwargs: Dict[str, Any] = Field(
40
        default={},
41
        description="Additional kwargs for the invoke_endpoint request.",
42
    )
43
    model_kwargs: Dict[str, Any] = Field(
44
        default={},
45
        description="kwargs to pass to the model.",
46
    )
47
    content_handler: BaseIOHandler = Field(
48
        default=DEFAULT_IO_HANDLER,
49
        description="used to serialize input, deserialize output, and remove a prefix.",
50
    )
51

52
    profile_name: Optional[str] = Field(
53
        description="The name of aws profile to use. If not given, then the default profile is used."
54
    )
55
    aws_access_key_id: Optional[str] = Field(description="AWS Access Key ID to use")
56
    aws_secret_access_key: Optional[str] = Field(
57
        description="AWS Secret Access Key to use"
58
    )
59
    aws_session_token: Optional[str] = Field(description="AWS Session Token to use")
60
    aws_region_name: Optional[str] = Field(
61
        description="AWS region name to use. Uses region configured in AWS CLI if not passed"
62
    )
63
    max_retries: Optional[int] = Field(
64
        default=3,
65
        description="The maximum number of API retries.",
66
        gte=0,
67
    )
68
    timeout: Optional[float] = Field(
69
        default=60.0,
70
        description="The timeout, in seconds, for API requests.",
71
        gte=0,
72
    )
73
    _client: Any = PrivateAttr()
74
    _completion_to_prompt: Callable[[str, Optional[str]], str] = PrivateAttr()
75

76
    def __init__(
77
        self,
78
        endpoint_name: str,
79
        endpoint_kwargs: Optional[Dict[str, Any]] = {},
80
        model_kwargs: Optional[Dict[str, Any]] = {},
81
        content_handler: Optional[BaseIOHandler] = DEFAULT_IO_HANDLER,
82
        profile_name: Optional[str] = None,
83
        aws_access_key_id: Optional[str] = None,
84
        aws_secret_access_key: Optional[str] = None,
85
        aws_session_token: Optional[str] = None,
86
        region_name: Optional[str] = None,
87
        max_retries: Optional[int] = 3,
88
        timeout: Optional[float] = 60.0,
89
        temperature: Optional[float] = 0.5,
90
        callback_manager: Optional[CallbackManager] = None,
91
        system_prompt: Optional[str] = None,
92
        messages_to_prompt: Optional[
93
            Callable[[Sequence[ChatMessage]], str]
94
        ] = LLAMA_MESSAGES_TO_PROMPT,
95
        completion_to_prompt: Callable[
96
            [str, Optional[str]], str
97
        ] = LLAMA_COMPLETION_TO_PROMPT,
98
        pydantic_program_mode: PydanticProgramMode = PydanticProgramMode.DEFAULT,
99
        output_parser: Optional[BaseOutputParser] = None,
100
        **kwargs: Any,
101
    ) -> None:
102
        if not endpoint_name:
103
            raise ValueError(
104
                "Missing required argument:`endpoint_name`"
105
                " Please specify the endpoint_name"
106
            )
107
        endpoint_kwargs = endpoint_kwargs or {}
108
        model_kwargs = model_kwargs or {}
109
        model_kwargs["temperature"] = temperature
110
        content_handler = content_handler
111
        self._completion_to_prompt = completion_to_prompt
112
        self._client = get_aws_service_client(
113
            service_name="sagemaker-runtime",
114
            profile_name=profile_name,
115
            region_name=region_name,
116
            aws_access_key_id=aws_access_key_id,
117
            aws_secret_access_key=aws_secret_access_key,
118
            aws_session_token=aws_session_token,
119
            max_retries=max_retries,
120
            timeout=timeout,
121
        )
122
        callback_manager = callback_manager or CallbackManager([])
123

124
        super().__init__(
125
            endpoint_name=endpoint_name,
126
            endpoint_kwargs=endpoint_kwargs,
127
            model_kwargs=model_kwargs,
128
            content_handler=content_handler,
129
            profile_name=profile_name,
130
            timeout=timeout,
131
            max_retries=max_retries,
132
            callback_manager=callback_manager,
133
            system_prompt=system_prompt,
134
            messages_to_prompt=messages_to_prompt,
135
            pydantic_program_mode=pydantic_program_mode,
136
            output_parser=output_parser,
137
        )
138

139
    @llm_completion_callback()
140
    def complete(
141
        self, prompt: str, formatted: bool = False, **kwargs: Any
142
    ) -> CompletionResponse:
143
        model_kwargs = {**self.model_kwargs, **kwargs}
144
        if not formatted:
145
            prompt = self._completion_to_prompt(prompt, self.system_prompt)
146

147
        request_body = self.content_handler.serialize_input(prompt, model_kwargs)
148
        response = self._client.invoke_endpoint(
149
            EndpointName=self.endpoint_name,
150
            Body=request_body,
151
            ContentType=self.content_handler.content_type,
152
            Accept=self.content_handler.accept,
153
            **self.endpoint_kwargs,
154
        )
155

156
        response["Body"] = self.content_handler.deserialize_output(response["Body"])
157
        text = self.content_handler.remove_prefix(response["Body"], prompt)
158

159
        return CompletionResponse(
160
            text=text,
161
            raw=response,
162
            additional_kwargs={
163
                "model_kwargs": model_kwargs,
164
                "endpoint_kwargs": self.endpoint_kwargs,
165
            },
166
        )
167

168
    @llm_completion_callback()
169
    def stream_complete(
170
        self, prompt: str, formatted: bool = False, **kwargs: Any
171
    ) -> CompletionResponseGen:
172
        model_kwargs = {**self.model_kwargs, **kwargs}
173
        if not formatted:
174
            prompt = self._completion_to_prompt(prompt, self.system_prompt)
175

176
        request_body = self.content_handler.serialize_input(prompt, model_kwargs)
177

178
        def gen() -> CompletionResponseGen:
179
            raw_text = ""
180
            prev_clean_text = ""
181
            for response in self._client.invoke_endpoint_with_response_stream(
182
                EndpointName=self.endpoint_name,
183
                Body=request_body,
184
                ContentType=self.content_handler.content_type,
185
                Accept=self.content_handler.accept,
186
                **self.endpoint_kwargs,
187
            )["Body"]:
188
                delta = self.content_handler.deserialize_streaming_output(
189
                    response["PayloadPart"]["Bytes"]
190
                )
191
                raw_text += delta
192
                clean_text = self.content_handler.remove_prefix(raw_text, prompt)
193
                delta = clean_text[len(prev_clean_text) :]
194
                prev_clean_text = clean_text
195

196
                yield CompletionResponse(text=clean_text, delta=delta, raw=response)
197

198
        return gen()
199

200
    @llm_chat_callback()
201
    def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> ChatResponse:
202
        prompt = self.messages_to_prompt(messages)
203
        completion_response = self.complete(prompt, formatted=True, **kwargs)
204
        return completion_response_to_chat_response(completion_response)
205

206
    @llm_chat_callback()
207
    def stream_chat(
208
        self, messages: Sequence[ChatMessage], **kwargs: Any
209
    ) -> ChatResponseGen:
210
        prompt = self.messages_to_prompt(messages)
211
        completion_response_gen = self.stream_complete(prompt, formatted=True, **kwargs)
212
        return stream_completion_response_to_chat_response(completion_response_gen)
213

214
    @llm_chat_callback()
215
    async def achat(
216
        self,
217
        messages: Sequence[ChatMessage],
218
        **kwargs: Any,
219
    ) -> ChatResponse:
220
        raise NotImplementedError
221

222
    @llm_chat_callback()
223
    async def astream_chat(
224
        self,
225
        messages: Sequence[ChatMessage],
226
        **kwargs: Any,
227
    ) -> ChatResponseAsyncGen:
228
        raise NotImplementedError
229

230
    @llm_completion_callback()
231
    async def acomplete(
232
        self, prompt: str, formatted: bool = False, **kwargs: Any
233
    ) -> CompletionResponse:
234
        raise NotImplementedError
235

236
    @llm_completion_callback()
237
    async def astream_complete(
238
        self, prompt: str, formatted: bool = False, **kwargs: Any
239
    ) -> CompletionResponseAsyncGen:
240
        raise NotImplementedError
241

242
    @classmethod
243
    def class_name(cls) -> str:
244
        return "SageMakerLLM"
245

246
    @property
247
    def metadata(self) -> LLMMetadata:
248
        """LLM metadata."""
249
        return LLMMetadata(
250
            model_name=self.endpoint_name,
251
        )
252

253

254
# Deprecated, kept for backwards compatibility
255
SageMakerLLMEndPoint = SageMakerLLM
256