From e629a33689f09ff5b3c0ce909e959921d4af759b Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 29 Apr 2024 19:06:04 -0700 Subject: [PATCH] review 1 --- ci/L0_backend_vllm/vllm_backend/test.sh | 5 +++++ ci/L0_multi_gpu/vllm_backend/test.sh | 2 +- ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/L0_backend_vllm/vllm_backend/test.sh b/ci/L0_backend_vllm/vllm_backend/test.sh index 2de00013..a6dd0aa7 100755 --- a/ci/L0_backend_vllm/vllm_backend/test.sh +++ b/ci/L0_backend_vllm/vllm_backend/test.sh @@ -50,6 +50,11 @@ function assert_curl_success { rm -rf models && mkdir -p models cp -r ${SAMPLE_MODELS_REPO}/vllm_model models/vllm_opt +# `vllm_opt`` model will be loaded on server start and stay loaded throughout +# unittesting. To test vllm model load/unload we use a dedicated +# `vllm_load_test`. To ensure that vllm's memory profiler will not error out +# on `vllm_load_test` load, we reduce "gpu_memory_utilization" for `vllm_opt`, +# so that at least 60% of GPU memory was available for other models. sed -i 's/"gpu_memory_utilization": 0.5/"gpu_memory_utilization": 0.4/' models/vllm_opt/1/model.json cp -r models/vllm_opt models/vllm_load_test diff --git a/ci/L0_multi_gpu/vllm_backend/test.sh b/ci/L0_multi_gpu/vllm_backend/test.sh index e891cd22..36369196 100755 --- a/ci/L0_multi_gpu/vllm_backend/test.sh +++ b/ci/L0_multi_gpu/vllm_backend/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py b/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py index 1d042f61..baa71632 100644 --- a/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py +++ b/ci/L0_multi_gpu/vllm_backend/vllm_multi_gpu_test.py @@ -1,4 +1,4 @@ -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions