review 1

triton-inference-server · Apr 30, 2024 · e629a33 · e629a33
1 parent 83b4fba
commit e629a33
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/ci/L0_backend_vllm/vllm_backend/test.sh b/ci/L0_backend_vllm/vllm_backend/test.sh
@@ -50,6 +50,11 @@ function assert_curl_success {
 
 rm -rf models && mkdir -p models
 cp -r ${SAMPLE_MODELS_REPO}/vllm_model models/vllm_opt
+# `vllm_opt`` model will be loaded on server start and stay loaded throughout
+# unittesting. To test vllm model load/unload we use a dedicated
+# `vllm_load_test`. To ensure that vllm's memory profiler will not error out
+# on `vllm_load_test` load, we reduce "gpu_memory_utilization" for `vllm_opt`,
+# so that at least 60% of GPU memory was available for other models.
 sed -i 's/"gpu_memory_utilization": 0.5/"gpu_memory_utilization": 0.4/' models/vllm_opt/1/model.json
 cp -r models/vllm_opt models/vllm_load_test
 

diff --git a/ci/L0_multi_gpu/vllm_backend/test.sh b/ci/L0_multi_gpu/vllm_backend/test.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py b/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py
@@ -1,4 +1,4 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions