ray-llm

1
# syntax=docker/dockerfile:1.4
2
# Note: TRTLLM backend is not included in the dockerfile, it is planned to be added in the future.
3

4
ARG RAY_IMAGE="anyscale/ray"
5
ARG RAY_TAG="2.9.0-py39-cu121"
6

7
# Use Anyscale base image
8
FROM ${RAY_IMAGE}:${RAY_TAG} AS aviary
9

10
ARG RAY_HOME="/home/ray"
11
ARG RAY_SITE_PACKAGES_DIR="${RAY_HOME}/anaconda3/lib/python3.9/site-packages"
12
ARG RAY_DIST_DIR="${RAY_HOME}/dist"
13
ARG RAY_MODELS_DIR="${RAY_HOME}/models"
14
ARG RAY_UID=1000
15
ARG RAY_GID=100
16

17
ENV RAY_SERVE_ENABLE_NEW_HANDLE_API=1
18
ENV RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1
19
ENV RAY_SERVE_ENABLE_JSON_LOGGING=1
20
ENV RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING=1
21
ENV RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S=310
22
ENV RAY_metrics_report_batch_size=400
23

24
ENV FORCE_CUDA=1
25
ENV HF_HUB_ENABLE_HF_TRANSFER=1
26
ENV SAFETENSORS_FAST_GPU=1
27
ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib:$LD_LIBRARY_PATH
28
ENV OMPI_ALLOW_RUN_AS_ROOT=1
29
ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
30

31
# Remove this line if we need the CUDA packages
32
# and NVIDIA fixes their repository #ir-gleaming-sky
33
RUN sudo rm -v /etc/apt/sources.list.d/cuda.list
34

35
# Install torch first
36
RUN pip install --no-cache-dir -U pip \
37
    && pip install --no-cache-dir -i https://download.pytorch.org/whl/cu121 torch~=2.1.0 torchvision torchaudio \
38
    && pip install --no-cache-dir tensorboard ninja
39

40
# The build context should be the root of the repo
41
# So this gives the model definitions
42
COPY --chown=${RAY_UID}:${RAY_GID} "./dist" "${RAY_DIST_DIR}"
43
COPY --chown=${RAY_UID}:${RAY_GID} "./models/continuous_batching" "${RAY_MODELS_DIR}/continuous_batching"
44
COPY --chown=${RAY_UID}:${RAY_GID} "./models/README.md" "${RAY_MODELS_DIR}/README.md"
45

46
# Install dependencies for aviary.
47
RUN cd "${RAY_DIST_DIR}" \
48
    # Update accelerate so transformers doesn't complain.
49
    && pip install --no-cache-dir -U accelerate \
50
    && pip install --no-cache-dir -U "$(ls rayllm-*.whl | head -n1)[frontend,backend]" \
51
    # Purge caches
52
    && pip cache purge || true \
53
    && conda clean -a \
54
    && rm -rf ~/.cache