llama-index

1
from typing import Any, Dict, Optional
2

3
from llama_index.legacy.bridge.pydantic import Field
4
from llama_index.legacy.constants import (
5
    DEFAULT_NUM_OUTPUTS,
6
    DEFAULT_TEMPERATURE,
7
)
8
from llama_index.legacy.core.llms.types import LLMMetadata
9
from llama_index.legacy.llms.generic_utils import get_from_param_or_env
10
from llama_index.legacy.llms.openai_like import OpenAILike
11

12
DEFAULT_API_BASE = "https://router.neutrinoapp.com/api/llm-router"
13
DEFAULT_ROUTER = "default"
14
MAX_CONTEXT_WINDOW = 200000
15

16

17
class Neutrino(OpenAILike):
18
    model: str = Field(
19
        description="The Neutrino router to use. See https://docs.neutrinoapp.com/router for details."
20
    )
21
    context_window: int = Field(
22
        default=MAX_CONTEXT_WINDOW,
23
        description="The maximum number of context tokens for the model. Defaults to the largest supported model (Claude).",
24
        gt=0,
25
    )
26
    is_chat_model: bool = Field(
27
        default=True,
28
        description=LLMMetadata.__fields__["is_chat_model"].field_info.description,
29
    )
30

31
    def __init__(
32
        self,
33
        model: Optional[str] = None,
34
        router: str = DEFAULT_ROUTER,
35
        temperature: float = DEFAULT_TEMPERATURE,
36
        max_tokens: int = DEFAULT_NUM_OUTPUTS,
37
        additional_kwargs: Optional[Dict[str, Any]] = None,
38
        max_retries: int = 5,
39
        api_base: Optional[str] = DEFAULT_API_BASE,
40
        api_key: Optional[str] = None,
41
        **kwargs: Any,
42
    ) -> None:
43
        additional_kwargs = additional_kwargs or {}
44

45
        api_base = get_from_param_or_env("api_base", api_base, "NEUTRINO_API_BASE")
46
        api_key = get_from_param_or_env("api_key", api_key, "NEUTRINO_API_KEY")
47

48
        model = model or router
49

50
        super().__init__(
51
            model=model,
52
            temperature=temperature,
53
            max_tokens=max_tokens,
54
            api_base=api_base,
55
            api_key=api_key,
56
            additional_kwargs=additional_kwargs,
57
            max_retries=max_retries,
58
            **kwargs,
59
        )
60

61
    @classmethod
62
    def class_name(cls) -> str:
63
        return "Neutrino_LLM"
64