feat: vllm increased customization

atoma-network · Dec 20, 2024 · 46cceda · 46cceda
1 parent bdd7d84
commit 46cceda
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/.env.example b/.env.example
@@ -52,7 +52,8 @@ CHAT_COMPLETIONS_MODEL=meta-llama/Llama-3.1-70B-Instruct
 CHAT_COMPLETIONS_MAX_MODEL_LEN=4096 # context length
 
 # vllm backend
-VLLM_TENSOR_PARALLEL_SIZE=1 # should be equal to GPU_COUNT
+# Know more about vllm engine arguments here: https://docs.vllm.ai/en/latest/usage/engine_args.html
+VLLM_ENGINE_ARGS=--model ${CHAT_COMPLETIONS_MODEL} --max-model-len ${CHAT_COMPLETIONS_MAX_MODEL_LEN}
 
 # ----------------------------------------------------------------------------------
 # embeddings server

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -131,7 +131,7 @@ services:
     ports:
       - "${CHAT_COMPLETIONS_SERVER_PORT}:8000"
     ipc: host
-    command: --model ${CHAT_COMPLETIONS_MODEL} --max-model-len ${CHAT_COMPLETIONS_MAX_MODEL_LEN} --tensor-parallel-size ${VLLM_TENSOR_PARALLEL_SIZE}
+    command: ${VLLM_ENGINE_ARGS}
 
   vllm-cpu:
     <<: *inference-service-cpu