.env.inference.example

# Development inference. Production must use immutable image digests and
# secret files with restrictive permissions.
LLAMACPP_SERVER_IMAGE=ghcr.io/ggml-org/llama.cpp:server
LLAMACPP_MODELS_DIR=./models/llamacpp
LLAMACPP_MODEL_FILE=model.gguf
LLAMACPP_SERVED_MODEL=modelforge-llamacpp
LLAMACPP_INFERENCE_API_KEY=replace-development-key
LLAMACPP_API_KEY_FILE=./secrets/llamacpp-api-key

VLLM_NVIDIA_IMAGE=vllm/vllm-openai:v0.25.1
VLLM_ROCM_IMAGE=vllm/vllm-openai-rocm:v0.25.1
# Host directory containing complete, pre-fetched and checksum-verified snapshots.
VLLM_MODELS_DIR=./models/vllm
# Container path to one snapshot below VLLM_MODELS_DIR; remote repository IDs are rejected.
VLLM_MODEL=/models/approved-snapshot
VLLM_SERVED_MODEL=modelforge-vllm
VLLM_GPU_MEMORY_UTILIZATION=0.85
VLLM_MAX_MODEL_LEN=4096
VLLM_INFERENCE_API_KEY=replace-development-key
VLLM_API_KEY_FILE=./secrets/vllm-api-key
MODELFORGE_INFERENCE_HOST_ALLOWLIST=llama-server,vllm-nvidia,vllm-rocm