ray-llm

1
deployment_config:
2
  autoscaling_config:
3
    min_replicas: 4
4
    initial_replicas: 4
5
    max_replicas: 10
6
    target_num_ongoing_requests_per_replica: 20
7
    metrics_interval_s: 10.0
8
    look_back_period_s: 30.0
9
    smoothing_factor: 0.5
10
    downscale_delay_s: 300.0
11
    upscale_delay_s: 15.0
12
  max_concurrent_queries: 48
13
  ray_actor_options:
14
    resources:
15
      mock_resource: 0
16
engine_config:
17
  model_id: VLLMFakeModel
18
  type: VLLMEngine
19
  max_total_tokens: 4096
20
  generation:
21
    prompt_format:
22
      system: "<<SYS>>\n{instruction}\n<</SYS>>\n\n"
23
      assistant: " {instruction} </s><s> "
24
      trailing_assistant: " "
25
      user: "[INST] {system}{instruction} [/INST]"
26
      system_in_user: true
27
      default_system_message: ""
28
    stopping_sequences: ["<unk>"]
29
scaling_config:
30
  num_workers: 1
31
  num_gpus_per_worker: 0
32
  num_cpus_per_worker: 1
33
  placement_strategy: "STRICT_PACK"
34
  resources_per_worker:
35
    mock_resource: 0
36