ray-llm

1
import json
2
import os
3

4
from ray import serve
5

6
from rayllm.backend.logger import get_logger
7
from rayllm.backend.server.routers.router_app import Router, router_app
8

9
logger = get_logger(__name__)
10

11

12
RouterDeployment = serve.deployment(
13
    route_prefix="/",
14
    # TODO make this configurable in aviary run
15
    autoscaling_config={
16
        "min_replicas": int(os.environ.get("AVIARY_ROUTER_MIN_REPLICAS", 2)),
17
        "initial_replicas": int(os.environ.get("AVIARY_ROUTER_INITIAL_REPLICAS", 2)),
18
        "max_replicas": int(os.environ.get("AVIARY_ROUTER_MAX_REPLICAS", 16)),
19
        "target_num_ongoing_requests_per_replica": int(
20
            os.environ.get("AVIARY_ROUTER_TARGET_NUM_ONGOING_REQUESTS_PER_REPLICA", 200)
21
        ),
22
    },
23
    ray_actor_options=json.loads(
24
        os.environ.get("AVIARY_ROUTER_RAY_ACTOR_OPTIONS", "{}")
25
    ),
26
    max_concurrent_queries=1000,  # Maximum backlog for a single replica
27
)(serve.ingress(router_app)(Router))
28