modelforge/compose.inference.prod.yml at main · voidstackloop/modelforge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: modelforge

services:
  server:
    networks: [default, inference]
    environment:
      MODELFORGE_INFERENCE_HOST_ALLOWLIST: ${MODELFORGE_INFERENCE_HOST_ALLOWLIST:-llama-server,vllm-nvidia,vllm-rocm}
    secrets:
      - llamacpp_inference_api_key
      - vllm_inference_api_key

  llama-server:
    profiles: ["inference-llamacpp"]
    image: ${LLAMACPP_SERVER_IMAGE:?set LLAMACPP_SERVER_IMAGE to an immutable tag or digest}
    command:
      - --model
      - /models/${LLAMACPP_MODEL_FILE:?set LLAMACPP_MODEL_FILE}
      - --alias
      - ${LLAMACPP_SERVED_MODEL:?set LLAMACPP_SERVED_MODEL}
      - --host
      - 0.0.0.0
      - --port
      - "8080"
      - --api-key-file
      - /run/secrets/llamacpp_inference_api_key
      - --metrics
      - --jinja
    secrets: [llamacpp_inference_api_key]
    volumes:
      - ${LLAMACPP_MODELS_DIR:?set LLAMACPP_MODELS_DIR}:/models:ro
    networks: [inference]
    read_only: true
    tmpfs: [/tmp:mode=1777]
    cap_drop: ["ALL"]
    security_opt: ["no-new-privileges:true"]
    healthcheck:
      test: ["CMD", "curl", "--fail", "--silent", "--show-error", "http://127.0.0.1:8080/health"]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 60s
    restart: unless-stopped

  vllm-nvidia:
    profiles: ["inference-vllm-nvidia"]
    image: ${VLLM_NVIDIA_IMAGE:?set VLLM_NVIDIA_IMAGE to an immutable tag or digest}
    entrypoint: ["/bin/sh", "-lc"]
    command:
      - >-
        case "$${VLLM_MODEL}" in /models/*) ;; *)
        echo "VLLM_MODEL must point to a verified snapshot under /models" >&2; exit 64;; esac;
        export VLLM_API_KEY="$$(cat /run/secrets/vllm_inference_api_key)";
        exec vllm serve "$${VLLM_MODEL}" --served-model-name "$${VLLM_SERVED_MODEL}"
        --host 0.0.0.0 --port 8000 --gpu-memory-utilization "$${VLLM_GPU_MEMORY_UTILIZATION}"
        --max-model-len "$${VLLM_MAX_MODEL_LEN}";
    environment:
      VLLM_MODEL: ${VLLM_MODEL:?set VLLM_MODEL to an approved local snapshot path}
      VLLM_SERVED_MODEL: ${VLLM_SERVED_MODEL:?set VLLM_SERVED_MODEL}
      VLLM_GPU_MEMORY_UTILIZATION: ${VLLM_GPU_MEMORY_UTILIZATION:-0.85}
      VLLM_MAX_MODEL_LEN: ${VLLM_MAX_MODEL_LEN:-4096}
    secrets: [vllm_inference_api_key]
    volumes:
      - ${VLLM_MODELS_DIR:?set VLLM_MODELS_DIR}:/models:ro
      - vllm-compile-cache:/home/vllm/.cache/vllm
    networks: [inference]
    shm_size: ${VLLM_SHM_SIZE:-8gb}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    cap_drop: ["ALL"]
    security_opt: ["no-new-privileges:true"]
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=5).read()"]
      interval: 10s
      timeout: 8s
      retries: 30
      start_period: 120s
    restart: unless-stopped

  vllm-rocm:
    profiles: ["inference-vllm-rocm"]
    image: ${VLLM_ROCM_IMAGE:?set VLLM_ROCM_IMAGE to an immutable tag or digest}
    entrypoint: ["/bin/sh", "-lc"]
    command:
      - >-
        case "$${VLLM_MODEL}" in /models/*) ;; *)
        echo "VLLM_MODEL must point to a verified snapshot under /models" >&2; exit 64;; esac;
        export VLLM_API_KEY="$$(cat /run/secrets/vllm_inference_api_key)";
        exec vllm serve "$${VLLM_MODEL}" --served-model-name "$${VLLM_SERVED_MODEL}"
        --host 0.0.0.0 --port 8000 --gpu-memory-utilization "$${VLLM_GPU_MEMORY_UTILIZATION}"
        --max-model-len "$${VLLM_MAX_MODEL_LEN}";
    environment:
      VLLM_MODEL: ${VLLM_MODEL:?set VLLM_MODEL to an approved local snapshot path}
      VLLM_SERVED_MODEL: ${VLLM_SERVED_MODEL:?set VLLM_SERVED_MODEL}
      VLLM_GPU_MEMORY_UTILIZATION: ${VLLM_GPU_MEMORY_UTILIZATION:-0.85}
      VLLM_MAX_MODEL_LEN: ${VLLM_MAX_MODEL_LEN:-4096}
    secrets: [vllm_inference_api_key]
    volumes:
      - ${VLLM_MODELS_DIR:?set VLLM_MODELS_DIR}:/models:ro
      - vllm-compile-cache:/home/vllm/.cache/vllm
    devices:
      - /dev/kfd:/dev/kfd
      - /dev/dri:/dev/dri
    group_add: ["video"]
    networks: [inference]
    shm_size: ${VLLM_SHM_SIZE:-8gb}
    cap_drop: ["ALL"]
    security_opt: ["no-new-privileges:true"]
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=5).read()"]
      interval: 10s
      timeout: 8s
      retries: 30
      start_period: 120s
    restart: unless-stopped

networks:
  inference:
    internal: true

volumes:
  vllm-compile-cache:

secrets:
  llamacpp_inference_api_key:
    file: ${LLAMACPP_API_KEY_FILE:?set LLAMACPP_API_KEY_FILE}
  vllm_inference_api_key:
    file: ${VLLM_API_KEY_FILE:?set VLLM_API_KEY_FILE}