From 65920c7328292d09c28e9ee8582f4c2b506dcf20 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Thu, 11 Jul 2024 15:00:07 +0000 Subject: [PATCH 01/10] Optuna L0 tests for single and ensemble models --- qa/L0_optuna_ensemble_model/check_results.py | 115 ++++++++++++++++++ qa/L0_optuna_ensemble_model/test.sh | 96 +++++++++++++++ .../test_config_generator.py | 64 ++++++++++ qa/L0_optuna_search/check_results.py | 114 +++++++++++++++++ qa/L0_optuna_search/test.sh | 93 ++++++++++++++ qa/L0_optuna_search/test_config_generator.py | 64 ++++++++++ 6 files changed, 546 insertions(+) create mode 100755 qa/L0_optuna_ensemble_model/check_results.py create mode 100755 qa/L0_optuna_ensemble_model/test.sh create mode 100755 qa/L0_optuna_ensemble_model/test_config_generator.py create mode 100755 qa/L0_optuna_search/check_results.py create mode 100755 qa/L0_optuna_search/test.sh create mode 100755 qa/L0_optuna_search/test_config_generator.py diff --git a/qa/L0_optuna_ensemble_model/check_results.py b/qa/L0_optuna_ensemble_model/check_results.py new file mode 100755 index 000000000..663484e32 --- /dev/null +++ b/qa/L0_optuna_ensemble_model/check_results.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +import yaml + + +class TestOutputValidator: + """ + Functions that validate the output + of the test + """ + + def __init__(self, config, test_name, analyzer_log): + self._config = config + self._models = config["profile_models"] + self._analyzer_log = analyzer_log + + check_function = self.__getattribute__(f"check_{test_name}") + + if check_function(): + sys.exit(0) + else: + sys.exit(1) + + def check_profile_logs(self): + """ + Check that each model was profiled the number of times + corresponding with batch size and concurrency combinations + + (No model config parameter combos expected here!) + """ + + with open(self._analyzer_log, "r") as f: + log_contents = f.read() + + # Model - ensemble_add_sub: + # concurrency: 1 to 1024 (11) + # Composing model - add: + # instance_group: 1 to 5 (5) + # Composing model - sub: + # instance_group: 1 to 5 (5) + # + # Minimum number of trials: 13 (5% of search space) + # Maximum number of trials: 27 (10% of search space) + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run + expected_min_num_measurements = 13 + 0 + expected_max_num_measurements = 27 + 36 + + for model in self._models: + token = f"Profiling {model}_config" + token_idx = 0 + found_count = 0 + while True: + token_idx = log_contents.find(token, token_idx + 1) + if token_idx == -1: + break + found_count += 1 + if ( + found_count < expected_min_num_measurements + or found_count > expected_max_num_measurements + ): + print( + f"\n***\n*** Expected range of measurements for {model} : {expected_min_num_measurements} to {expected_max_num_measurements}. " + f"Found {found_count}. \n***" + ) + return False + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-f", + "--config-file", + type=str, + required=True, + help="The path to the config yaml file.", + ) + parser.add_argument( + "-l", + "--analyzer-log-file", + type=str, + required=True, + help="The full path to the analyzer log.", + ) + parser.add_argument( + "-t", + "--test-name", + type=str, + required=True, + help="The name of the test to be run.", + ) + args = parser.parse_args() + + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) + + TestOutputValidator(config, args.test_name, args.analyzer_log_file) diff --git a/qa/L0_optuna_ensemble_model/test.sh b/qa/L0_optuna_ensemble_model/test.sh new file mode 100755 index 000000000..ece65cd23 --- /dev/null +++ b/qa/L0_optuna_ensemble_model/test.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ../common/util.sh +source ../common/check_analyzer_results.sh +create_logs_dir "L0_ensemble_model" + +# Set test parameters +MODEL_ANALYZER="$(which model-analyzer)" +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +MODEL_REPOSITORY=${MODEL_REPOSITORY:="/opt/triton-model-analyzer/examples/quick-start"} +QA_MODELS="ensemble_add_sub" +MODEL_NAMES="$(echo $QA_MODELS | sed 's/ /,/g')" +TRITON_LAUNCH_MODE=${TRITON_LAUNCH_MODE:="local"} +CLIENT_PROTOCOL="grpc" +PORTS=($(find_available_ports 3)) +GPUS=($(get_all_gpus_uuids)) +OUTPUT_MODEL_REPOSITORY=${OUTPUT_MODEL_REPOSITORY:=$(get_output_directory)} +CONFIG_FILE="config.yml" +FILENAME_SERVER_ONLY="server-metrics.csv" +FILENAME_INFERENCE_MODEL="model-metrics-inference.csv" +FILENAME_GPU_MODEL="model-metrics-gpu.csv" + +mkdir -p $MODEL_REPOSITORY/ensemble_add_sub/1 +rm -rf $OUTPUT_MODEL_REPOSITORY +create_result_paths +SERVER_LOG=$TEST_LOG_DIR/server.log + +python3 test_config_generator.py --profile-models $MODEL_NAMES + +# Run the analyzer and check the results +RET=0 + +set +e + +MODEL_ANALYZER_ARGS="-m $MODEL_REPOSITORY -f $CONFIG_FILE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --client-protocol=$CLIENT_PROTOCOL --triton-launch-mode=$TRITON_LAUNCH_MODE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-http-endpoint localhost:${PORTS[0]} --triton-grpc-endpoint localhost:${PORTS[1]}" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:${PORTS[2]}/metrics" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-detailed-reports --triton-output-path=$SERVER_LOG" +MODEL_ANALYZER_SUBCOMMAND="profile" + +run_analyzer + +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" + cat $ANALYZER_LOG + RET=1 +else + # Check the Analyzer log for correct output + TEST_NAME='profile_logs' + python3 check_results.py -f $CONFIG_FILE -t $TEST_NAME -l $ANALYZER_LOG + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed for $TEST_NAME test.\n***" + cat $ANALYZER_LOG + RET=1 + fi + + SERVER_METRICS_FILE=${EXPORT_PATH}/results/${FILENAME_SERVER_ONLY} + MODEL_METRICS_GPU_FILE=${EXPORT_PATH}/results/${FILENAME_GPU_MODEL} + MODEL_METRICS_INFERENCE_FILE=${EXPORT_PATH}/results/${FILENAME_INFERENCE_MODEL} + + for file in SERVER_METRICS_FILE, MODEL_METRICS_GPU_FILE, MODEL_METRICS_INFERENCE_FILE; do + check_no_csv_exists $file + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed.\n***" + cat $ANALYZER_LOG + RET=1 + fi + done +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test PASSED\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/L0_optuna_ensemble_model/test_config_generator.py b/qa/L0_optuna_ensemble_model/test_config_generator.py new file mode 100755 index 000000000..15125d425 --- /dev/null +++ b/qa/L0_optuna_ensemble_model/test_config_generator.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import yaml + + +class TestConfigGenerator: + """ + This class contains functions that + create configs for various test scenarios. + + The `setup` function does the work common to all tests + + TO ADD A TEST: Simply add a member function whose name starts + with 'generate'. + """ + + def __init__(self): + test_functions = [ + self.__getattribute__(name) + for name in dir(self) + if name.startswith("generate") + ] + + for test_function in test_functions: + self.setup() + test_function() + + def setup(self): + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", + "--profile-models", + type=str, + required=True, + help="Comma separated list of models to be profiled", + ) + + args = parser.parse_args() + self.config = {} + self.config["profile_models"] = sorted(args.profile_models.split(",")) + + def generate_config(self): + with open("config.yml", "w+") as f: + yaml.dump(self.config, f) + + +if __name__ == "__main__": + TestConfigGenerator() diff --git a/qa/L0_optuna_search/check_results.py b/qa/L0_optuna_search/check_results.py new file mode 100755 index 000000000..9e3bcaf0e --- /dev/null +++ b/qa/L0_optuna_search/check_results.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +import yaml + + +class TestOutputValidator: + """ + Functions that validate the output + of the test + """ + + def __init__(self, config, test_name, analyzer_log): + self._config = config + self._models = config["profile_models"] + self._analyzer_log = analyzer_log + + check_function = self.__getattribute__(f"check_{test_name}") + + if check_function(): + sys.exit(0) + else: + sys.exit(1) + + def check_profile_logs(self): + """ + Check that each model was profiled the number of times + corresponding with batch size and concurrency combinations + + (No model config parameter combos expected here!) + """ + + with open(self._analyzer_log, "r") as f: + log_contents = f.read() + + # Number of configs in search space: 440 + # Model - add_sub: + # concurrency: 1 to 1024 (11) + # max_batch_size: 1 to 128 (8) + # instance_group: 1 to 5 (5) + # + # Minimum number of trials: 22 (5% of search space) + # Maximum number of trials: 44 (10% of search space) + # + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run + expected_min_num_measurements = 22 + expected_max_num_measurements = 44 + 36 + for model in self._models: + token = f"Profiling {model}_config" + token_idx = 0 + found_count = 0 + while True: + token_idx = log_contents.find(token, token_idx + 1) + if token_idx == -1: + break + found_count += 1 + if ( + found_count < expected_min_num_measurements + or found_count > expected_max_num_measurements + ): + print( + f"\n***\n*** Expected range of measurements for {model} : {expected_min_num_measurements} to {expected_max_num_measurements}. " + f"Found {found_count}. \n***" + ) + return False + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-f", + "--config-file", + type=str, + required=True, + help="The path to the config yaml file.", + ) + parser.add_argument( + "-l", + "--analyzer-log-file", + type=str, + required=True, + help="The full path to the analyzer log.", + ) + parser.add_argument( + "-t", + "--test-name", + type=str, + required=True, + help="The name of the test to be run.", + ) + args = parser.parse_args() + + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) + + TestOutputValidator(config, args.test_name, args.analyzer_log_file) diff --git a/qa/L0_optuna_search/test.sh b/qa/L0_optuna_search/test.sh new file mode 100755 index 000000000..be107521a --- /dev/null +++ b/qa/L0_optuna_search/test.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ../common/util.sh +source ../common/check_analyzer_results.sh +create_logs_dir "L0_optuna_search" + +# Set test parameters +MODEL_ANALYZER="`which model-analyzer`" +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +MODEL_REPOSITORY=${MODEL_REPOSITORY:="/opt/triton-model-analyzer/examples/quick-start"} +QA_MODELS="add_sub" +MODEL_NAMES="$(echo $QA_MODELS | sed 's/ /,/g')" +TRITON_LAUNCH_MODE=${TRITON_LAUNCH_MODE:="local"} +CLIENT_PROTOCOL="grpc" +PORTS=(`find_available_ports 3`) +GPUS=(`get_all_gpus_uuids`) +OUTPUT_MODEL_REPOSITORY=${OUTPUT_MODEL_REPOSITORY:=`get_output_directory`} +CONFIG_FILE="config.yml" +FILENAME_SERVER_ONLY="server-metrics.csv" +FILENAME_INFERENCE_MODEL="model-metrics-inference.csv" +FILENAME_GPU_MODEL="model-metrics-gpu.csv" + +rm -rf $OUTPUT_MODEL_REPOSITORY +create_result_paths + +python3 test_config_generator.py --profile-models $MODEL_NAMES + +# Run the analyzer and check the results +RET=0 + +set +e + +MODEL_ANALYZER_ARGS="-m $MODEL_REPOSITORY -f $CONFIG_FILE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --client-protocol=$CLIENT_PROTOCOL --triton-launch-mode=$TRITON_LAUNCH_MODE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-http-endpoint localhost:${PORTS[0]} --triton-grpc-endpoint localhost:${PORTS[1]}" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:${PORTS[2]}/metrics" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --latency-budget 10" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-summary-reports" +MODEL_ANALYZER_SUBCOMMAND="profile" +run_analyzer +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" + cat $ANALYZER_LOG + RET=1 +else + # Check the Analyzer log for correct output + TEST_NAME='profile_logs' + python3 check_results.py -f $CONFIG_FILE -t $TEST_NAME -l $ANALYZER_LOG + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed for $TEST_NAME test.\n***" + cat $ANALYZER_LOG + RET=1 + fi + + SERVER_METRICS_FILE=${EXPORT_PATH}/results/${FILENAME_SERVER_ONLY} + MODEL_METRICS_GPU_FILE=${EXPORT_PATH}/results/${FILENAME_GPU_MODEL} + MODEL_METRICS_INFERENCE_FILE=${EXPORT_PATH}/results/${FILENAME_INFERENCE_MODEL} + + for file in SERVER_METRICS_FILE, MODEL_METRICS_GPU_FILE, MODEL_METRICS_INFERENCE_FILE; do + check_no_csv_exists $file + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed.\n***" + cat $ANALYZER_LOG + RET=1 + fi + done +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test PASSED\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/L0_optuna_search/test_config_generator.py b/qa/L0_optuna_search/test_config_generator.py new file mode 100755 index 000000000..f3bdb7a19 --- /dev/null +++ b/qa/L0_optuna_search/test_config_generator.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import yaml + + +class TestConfigGenerator: + """ + This class contains functions that + create configs for various test scenarios. + + The `setup` function does the work common to all tests + + TO ADD A TEST: Simply add a member function whose name starts + with 'generate'. + """ + + def __init__(self): + test_functions = [ + self.__getattribute__(name) + for name in dir(self) + if name.startswith("generate") + ] + + for test_function in test_functions: + self.setup() + test_function() + + def setup(self): + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", + "--profile-models", + type=str, + required=True, + help="The config file for this test", + ) + + args = parser.parse_args() + self.config = {} + self.config["profile_models"] = sorted(args.profile_models.split(",")) + + def generate_config(self): + with open("config.yml", "w+") as f: + yaml.dump(self.config, f) + + +if __name__ == "__main__": + TestConfigGenerator() From b0d3956afedb39c8c5bfd52cec89a0830e1d3ac2 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Thu, 11 Jul 2024 19:01:13 +0000 Subject: [PATCH 02/10] Adding BLS test --- qa/L0_optuna_bls_model/check_results.py | 115 ++++++++++++++++++ qa/L0_optuna_bls_model/input_data.json | 21 ++++ qa/L0_optuna_bls_model/test.sh | 97 +++++++++++++++ .../test_config_generator.py | 81 ++++++++++++ qa/L0_optuna_ensemble_model/check_results.py | 1 + 5 files changed, 315 insertions(+) create mode 100755 qa/L0_optuna_bls_model/check_results.py create mode 100644 qa/L0_optuna_bls_model/input_data.json create mode 100755 qa/L0_optuna_bls_model/test.sh create mode 100755 qa/L0_optuna_bls_model/test_config_generator.py diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py new file mode 100755 index 000000000..2a7577df2 --- /dev/null +++ b/qa/L0_optuna_bls_model/check_results.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +import yaml + + +class TestOutputValidator: + """ + Functions that validate the output + of the test + """ + + def __init__(self, config, test_name, analyzer_log): + self._config = config + self._models = config["profile_models"] + self._analyzer_log = analyzer_log + + check_function = self.__getattribute__(f"check_{test_name}") + + if check_function(): + sys.exit(0) + else: + sys.exit(1) + + def check_profile_logs(self): + """ + Check that each model was profiled the number of times + corresponding with batch size and concurrency combinations + + (No model config parameter combos expected here!) + """ + + with open(self._analyzer_log, "r") as f: + log_contents = f.read() + + # Number of configs in search space: 25 + # Model - bls: + # instance_group: 1 to 5 (5) + # Composing model - add: + # instance_group: 1 to 5 (5) + # + # Minimum number of trials: 10 (40% of search space) + # Maximum number of trials: 20 (80% of search space) + # + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run + expected_min_num_measurements = 10 + 0 + expected_max_num_measurements = 20 + 36 + + for model in self._models: + token = f"Profiling {model}_config" + token_idx = 0 + found_count = 0 + while True: + token_idx = log_contents.find(token, token_idx + 1) + if token_idx == -1: + break + found_count += 1 + if ( + found_count < expected_min_num_measurements + or found_count > expected_max_num_measurements + ): + print( + f"\n***\n*** Expected range of measurements for {model} : {expected_min_num_measurements} to {expected_max_num_measurements}. " + f"Found {found_count}. \n***" + ) + return False + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-f", + "--config-file", + type=str, + required=True, + help="The path to the config yaml file.", + ) + parser.add_argument( + "-l", + "--analyzer-log-file", + type=str, + required=True, + help="The full path to the analyzer log.", + ) + parser.add_argument( + "-t", + "--test-name", + type=str, + required=True, + help="The name of the test to be run.", + ) + args = parser.parse_args() + + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) + + TestOutputValidator(config, args.test_name, args.analyzer_log_file) diff --git a/qa/L0_optuna_bls_model/input_data.json b/qa/L0_optuna_bls_model/input_data.json new file mode 100644 index 000000000..505e15ce5 --- /dev/null +++ b/qa/L0_optuna_bls_model/input_data.json @@ -0,0 +1,21 @@ +{ + "data": [ + { + "MODEL_NAME": [ + "add" + ], + "INPUT0": [ + 0.74106514, + 0.7371813, + 0.5274665, + 0.13930903 + ], + "INPUT1": [ + 0.7845891, + 0.88089234, + 0.8466405, + 0.55024815 + ] + } + ] +} diff --git a/qa/L0_optuna_bls_model/test.sh b/qa/L0_optuna_bls_model/test.sh new file mode 100755 index 000000000..542782d47 --- /dev/null +++ b/qa/L0_optuna_bls_model/test.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ../common/util.sh +source ../common/check_analyzer_results.sh +create_logs_dir "L0_bls_model" + +# Set test parameters +MODEL_ANALYZER="$(which model-analyzer)" +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +MODEL_REPOSITORY=${MODEL_REPOSITORY:="/opt/triton-model-analyzer/examples/quick-start"} +QA_MODELS="bls" +BLS_COMPOSING_MODELS="add" +INPUT_JSON="$(pwd)/input_data.json" +MODEL_NAMES="$(echo $QA_MODELS | sed 's/ /,/g')" +TRITON_LAUNCH_MODE=${TRITON_LAUNCH_MODE:="local"} +CLIENT_PROTOCOL="grpc" +PORTS=($(find_available_ports 3)) +GPUS=($(get_all_gpus_uuids)) +OUTPUT_MODEL_REPOSITORY=${OUTPUT_MODEL_REPOSITORY:=$(get_output_directory)} +CONFIG_FILE="config.yml" +FILENAME_SERVER_ONLY="server-metrics.csv" +FILENAME_INFERENCE_MODEL="model-metrics-inference.csv" +FILENAME_GPU_MODEL="model-metrics-gpu.csv" + +rm -rf $OUTPUT_MODEL_REPOSITORY +create_result_paths +SERVER_LOG=$TEST_LOG_DIR/server.log + +python3 test_config_generator.py --profile-models $MODEL_NAMES --bls-composing-models $BLS_COMPOSING_MODELS -i $INPUT_JSON + +# Run the analyzer and check the results +RET=0 + +set +e + +MODEL_ANALYZER_ARGS="-m $MODEL_REPOSITORY -f $CONFIG_FILE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --client-protocol=$CLIENT_PROTOCOL --triton-launch-mode=$TRITON_LAUNCH_MODE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-http-endpoint localhost:${PORTS[0]} --triton-grpc-endpoint localhost:${PORTS[1]}" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:${PORTS[2]}/metrics" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --min-percentage-of-search-space 40 --max-percentage-of-search-space 80" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-detailed-reports --triton-output-path=$SERVER_LOG" +MODEL_ANALYZER_SUBCOMMAND="profile" + +run_analyzer + +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" + cat $ANALYZER_LOG + RET=1 +else + # Check the Analyzer log for correct output + TEST_NAME='profile_logs' + python3 check_results.py -f $CONFIG_FILE -t $TEST_NAME -l $ANALYZER_LOG + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed for $TEST_NAME test.\n***" + cat $ANALYZER_LOG + RET=1 + fi + + SERVER_METRICS_FILE=${EXPORT_PATH}/results/${FILENAME_SERVER_ONLY} + MODEL_METRICS_GPU_FILE=${EXPORT_PATH}/results/${FILENAME_GPU_MODEL} + MODEL_METRICS_INFERENCE_FILE=${EXPORT_PATH}/results/${FILENAME_INFERENCE_MODEL} + + for file in SERVER_METRICS_FILE, MODEL_METRICS_GPU_FILE, MODEL_METRICS_INFERENCE_FILE; do + check_no_csv_exists $file + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed.\n***" + cat $ANALYZER_LOG + RET=1 + fi + done +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test PASSED\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/L0_optuna_bls_model/test_config_generator.py b/qa/L0_optuna_bls_model/test_config_generator.py new file mode 100755 index 000000000..67ab0edbe --- /dev/null +++ b/qa/L0_optuna_bls_model/test_config_generator.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import yaml + + +class TestConfigGenerator: + """ + This class contains functions that + create configs for various test scenarios. + + The `setup` function does the work common to all tests + + TO ADD A TEST: Simply add a member function whose name starts + with 'generate'. + """ + + def __init__(self): + test_functions = [ + self.__getattribute__(name) + for name in dir(self) + if name.startswith("generate") + ] + + for test_function in test_functions: + self.setup() + test_function() + + def setup(self): + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", + "--profile-models", + type=str, + required=True, + help="Comma separated list of models to be profiled", + ) + parser.add_argument( + "--bls-composing-models", + type=str, + required=True, + help="Comma separated list of BLS composing models", + ) + parser.add_argument( + "-i", + "--input-json-file", + type=str, + required=True, + help="Input data JSON file path", + ) + + args = parser.parse_args() + self.config = {} + self.config["profile_models"] = sorted(args.profile_models.split(",")) + self.config["bls_composing_models"] = sorted( + args.bls_composing_models.split(",") + ) + self.config["perf_analyzer_flags"] = {"input-data": args.input_json_file} + + def generate_config(self): + with open("config.yml", "w+") as f: + yaml.dump(self.config, f) + + +if __name__ == "__main__": + TestConfigGenerator() diff --git a/qa/L0_optuna_ensemble_model/check_results.py b/qa/L0_optuna_ensemble_model/check_results.py index 663484e32..798afa70b 100755 --- a/qa/L0_optuna_ensemble_model/check_results.py +++ b/qa/L0_optuna_ensemble_model/check_results.py @@ -58,6 +58,7 @@ def check_profile_logs(self): # # Minimum number of trials: 13 (5% of search space) # Maximum number of trials: 27 (10% of search space) + # # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default # 0 because all concurrencies could have been tested during the optuna run expected_min_num_measurements = 13 + 0 From a313d9b8542db5b5c91bbdff6218b96465bcc830 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Thu, 11 Jul 2024 19:15:45 +0000 Subject: [PATCH 03/10] Adding L0 multi-model optuna test --- qa/L0_optuna_multi_model/check_results.py | 117 ++++++++++++++++++ qa/L0_optuna_multi_model/test.sh | 92 ++++++++++++++ .../test_config_generator.py | 64 ++++++++++ 3 files changed, 273 insertions(+) create mode 100755 qa/L0_optuna_multi_model/check_results.py create mode 100755 qa/L0_optuna_multi_model/test.sh create mode 100755 qa/L0_optuna_multi_model/test_config_generator.py diff --git a/qa/L0_optuna_multi_model/check_results.py b/qa/L0_optuna_multi_model/check_results.py new file mode 100755 index 000000000..79cfceed4 --- /dev/null +++ b/qa/L0_optuna_multi_model/check_results.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +import yaml + + +class TestOutputValidator: + """ + Functions that validate the output + of the test + """ + + def __init__(self, config, test_name, analyzer_log): + self._config = config + self._models = config["profile_models"] + self._analyzer_log = analyzer_log + + check_function = self.__getattribute__(f"check_{test_name}") + + if check_function(): + sys.exit(0) + else: + sys.exit(1) + + def check_profile_logs(self): + """ + Check that each model was profiled the number of times + corresponding with batch size and concurrency combinations + + (No model config parameter combos expected here!) + """ + + with open(self._analyzer_log, "r") as f: + log_contents = f.read() + + # Number of configs in search space: 1600 + # Model - add_sub: + # max_batch_size: 1 to 128 (8) + # instance_group: 1 to 5 (5) + # Model - vgg19_libtorch: + # max_batch_size: 1 to 128 (8) + # instance_group: 1 to 5 (5) + # + # Minimum number of trials: 32 (2% of search space) + # Maximum number of trials: 64 (4% of search space) + # + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run + + expected_min_num_measurements = 32 + 0 + expected_max_num_measurements = 64 + 36 + for model in self._models: + token = f"Profiling {model}_config" + token_idx = 0 + found_count = 0 + while True: + token_idx = log_contents.find(token, token_idx + 1) + if token_idx == -1: + break + found_count += 1 + if ( + found_count < expected_min_num_measurements + or found_count > expected_max_num_measurements + ): + print( + f"\n***\n*** Expected range of measurements for {model} : {expected_min_num_measurements} to {expected_max_num_measurements}. " + f"Found {found_count}. \n***" + ) + return False + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-f", + "--config-file", + type=str, + required=True, + help="The path to the config yaml file.", + ) + parser.add_argument( + "-l", + "--analyzer-log-file", + type=str, + required=True, + help="The full path to the analyzer log.", + ) + parser.add_argument( + "-t", + "--test-name", + type=str, + required=True, + help="The name of the test to be run.", + ) + args = parser.parse_args() + + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) + + TestOutputValidator(config, args.test_name, args.analyzer_log_file) diff --git a/qa/L0_optuna_multi_model/test.sh b/qa/L0_optuna_multi_model/test.sh new file mode 100755 index 000000000..121c525d6 --- /dev/null +++ b/qa/L0_optuna_multi_model/test.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ../common/util.sh +source ../common/check_analyzer_results.sh +create_logs_dir "L0_quick_search_multi_model" + +# Set test parameters +MODEL_ANALYZER="`which model-analyzer`" +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +MODEL_REPOSITORY=${MODEL_REPOSITORY:="/opt/triton-model-analyzer/examples/quick-start"} +QA_MODELS="add,sub" +MODEL_NAMES="$(echo $QA_MODELS | sed 's/ /,/g')" +TRITON_LAUNCH_MODE=${TRITON_LAUNCH_MODE:="local"} +CLIENT_PROTOCOL="grpc" +PORTS=(`find_available_ports 3`) +GPUS=(`get_all_gpus_uuids`) +OUTPUT_MODEL_REPOSITORY=${OUTPUT_MODEL_REPOSITORY:=`get_output_directory`} +CONFIG_FILE="config.yml" +FILENAME_SERVER_ONLY="server-metrics.csv" +FILENAME_INFERENCE_MODEL="model-metrics-inference.csv" +FILENAME_GPU_MODEL="model-metrics-gpu.csv" + +create_result_paths +rm -rf $OUTPUT_MODEL_REPOSITORY + +python3 test_config_generator.py --profile-models $MODEL_NAMES + +# Run the analyzer and check the results +RET=0 + +set +e + +MODEL_ANALYZER_ARGS="-m $MODEL_REPOSITORY -f $CONFIG_FILE" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --client-protocol=$CLIENT_PROTOCOL --triton-launch-mode=$TRITON_LAUNCH_MODE --run-config-profile-models-concurrently-enable" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-http-endpoint localhost:${PORTS[0]} --triton-grpc-endpoint localhost:${PORTS[1]}" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost:${PORTS[2]}/metrics" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --skip-summary-reports" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --min-percentage-search-space 2 --max-percentage-search-space 4" +MODEL_ANALYZER_SUBCOMMAND="profile" +run_analyzer +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" + cat $ANALYZER_LOG + RET=1 +else + # Check the Analyzer log for correct output + TEST_NAME='profile_logs' + python3 check_results.py -f $CONFIG_FILE -t $TEST_NAME -l $ANALYZER_LOG + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed for $TEST_NAME test.\n***" + cat $ANALYZER_LOG + RET=1 + fi + + SERVER_METRICS_FILE=${EXPORT_PATH}/results/${FILENAME_SERVER_ONLY} + MODEL_METRICS_GPU_FILE=${EXPORT_PATH}/results/${FILENAME_GPU_MODEL} + MODEL_METRICS_INFERENCE_FILE=${EXPORT_PATH}/results/${FILENAME_INFERENCE_MODEL} + + for file in SERVER_METRICS_FILE, MODEL_METRICS_GPU_FILE, MODEL_METRICS_INFERENCE_FILE; do + check_no_csv_exists $file + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Output Verification Failed.\n***" + cat $ANALYZER_LOG + RET=1 + fi + done +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test PASSED\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/L0_optuna_multi_model/test_config_generator.py b/qa/L0_optuna_multi_model/test_config_generator.py new file mode 100755 index 000000000..f3bdb7a19 --- /dev/null +++ b/qa/L0_optuna_multi_model/test_config_generator.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import yaml + + +class TestConfigGenerator: + """ + This class contains functions that + create configs for various test scenarios. + + The `setup` function does the work common to all tests + + TO ADD A TEST: Simply add a member function whose name starts + with 'generate'. + """ + + def __init__(self): + test_functions = [ + self.__getattribute__(name) + for name in dir(self) + if name.startswith("generate") + ] + + for test_function in test_functions: + self.setup() + test_function() + + def setup(self): + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", + "--profile-models", + type=str, + required=True, + help="The config file for this test", + ) + + args = parser.parse_args() + self.config = {} + self.config["profile_models"] = sorted(args.profile_models.split(",")) + + def generate_config(self): + with open("config.yml", "w+") as f: + yaml.dump(self.config, f) + + +if __name__ == "__main__": + TestConfigGenerator() From 98bc53e01bbad37bf0a5398b6e1bf43305e998bf Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Thu, 11 Jul 2024 20:28:40 +0000 Subject: [PATCH 04/10] Fixing multi-model test --- qa/L0_optuna_multi_model/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/L0_optuna_multi_model/test.sh b/qa/L0_optuna_multi_model/test.sh index 121c525d6..909397984 100755 --- a/qa/L0_optuna_multi_model/test.sh +++ b/qa/L0_optuna_multi_model/test.sh @@ -15,7 +15,7 @@ source ../common/util.sh source ../common/check_analyzer_results.sh -create_logs_dir "L0_quick_search_multi_model" +create_logs_dir "L0_optuna_multi_model" # Set test parameters MODEL_ANALYZER="`which model-analyzer`" @@ -51,7 +51,7 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --skip-summary-reports" -MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --min-percentage-search-space 2 --max-percentage-search-space 4" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --min-percentage-of-search-space 2 --max-percentage-of-search-space 4" MODEL_ANALYZER_SUBCOMMAND="profile" run_analyzer if [ $? -ne 0 ]; then From 7b6080ef1898dbb1881e09220d75f64d5b2de596 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Fri, 12 Jul 2024 00:11:49 +0000 Subject: [PATCH 05/10] Fixing CLI bugs --- .../config/input/config_command_profile.py | 10 +++++----- tests/test_cli.py | 10 +++++----- tests/test_optuna_run_config_generator.py | 12 ++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/model_analyzer/config/input/config_command_profile.py b/model_analyzer/config/input/config_command_profile.py index aabb2b1f5..7b2b557b6 100755 --- a/model_analyzer/config/input/config_command_profile.py +++ b/model_analyzer/config/input/config_command_profile.py @@ -927,7 +927,7 @@ def _add_run_search_configs(self): self._add_config( ConfigField( "min_percentage_of_search_space", - flags=["--min_percentage_of_search_space"], + flags=["--min-percentage-of-search-space"], field_type=ConfigPrimitive(int), default_value=DEFAULT_OPTUNA_MIN_PERCENTAGE_OF_SEARCH_SPACE, description="Minimum percentage of the search space to profile when using Optuna", @@ -936,7 +936,7 @@ def _add_run_search_configs(self): self._add_config( ConfigField( "max_percentage_of_search_space", - flags=["--max_percentage_of_search_space"], + flags=["--max-percentage-of-search-space"], field_type=ConfigPrimitive(int), default_value=DEFAULT_OPTUNA_MAX_PERCENTAGE_OF_SEARCH_SPACE, description="Maximum percentage of the search space to profile when using Optuna", @@ -945,7 +945,7 @@ def _add_run_search_configs(self): self._add_config( ConfigField( "optuna_min_trials", - flags=["--optuna_min_trials"], + flags=["--optuna-min-trials"], field_type=ConfigPrimitive(int), default_value=DEFAULT_OPTUNA_MIN_TRIALS, description="Minimum number of trials to profile when using Optuna", @@ -954,7 +954,7 @@ def _add_run_search_configs(self): self._add_config( ConfigField( "optuna_max_trials", - flags=["--optuna_max_trials"], + flags=["--optuna-max-trials"], field_type=ConfigPrimitive(int), default_value=DEFAULT_OPTUNA_MAX_TRIALS, description="Maximum number of trials to profile when using Optuna", @@ -963,7 +963,7 @@ def _add_run_search_configs(self): self._add_config( ConfigField( "optuna_early_exit_threshold", - flags=["--optuna_early_exit_threshold"], + flags=["--optuna-early-exit-threshold"], field_type=ConfigPrimitive(int), default_value=DEFAULT_OPTUNA_EARLY_EXIT_THRESHOLD, description="Number of trials without improvement before triggering early exit when using Optuna", diff --git a/tests/test_cli.py b/tests/test_cli.py index 68290bc23..b0460d88c 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -88,11 +88,11 @@ def get_test_options(): OptionStruct("int", "profile", "--run-config-search-min-instance-count", None, "2", "1"), OptionStruct("int", "profile", "--run-config-search-max-instance-count", None, "10", "5"), OptionStruct("int", "profile", "--run-config-search-max-binary-search-steps", None, "10", "5"), - OptionStruct("int", "profile", "--min_percentage_of_search_space", None, "10", "5"), - OptionStruct("int", "profile", "--max_percentage_of_search_space", None, "5", "10"), - OptionStruct("int", "profile", "--optuna_min_trials", None, "10", "20"), - OptionStruct("int", "profile", "--optuna_max_trials", None, "5", "200"), - OptionStruct("int", "profile", "--optuna_early_exit_threshold", None, "5", "10"), + OptionStruct("int", "profile", "--min-percentage-of-search-space", None, "10", "5"), + OptionStruct("int", "profile", "--max-percentage-of-search-space", None, "5", "10"), + OptionStruct("int", "profile", "--optuna-min-trials", None, "10", "20"), + OptionStruct("int", "profile", "--optuna-max-trials", None, "5", "200"), + OptionStruct("int", "profile", "--optuna-early-exit-threshold", None, "5", "10"), OptionStruct("float", "profile", "--monitoring-interval", "-i", "10.0", "1.0"), OptionStruct("float", "profile", "--perf-analyzer-cpu-util", None, "10.0", str(psutil.cpu_count() * 80.0)), OptionStruct("int", "profile", "--num-configs-per-model", None, "10", "3"), diff --git a/tests/test_optuna_run_config_generator.py b/tests/test_optuna_run_config_generator.py index 01953d1fe..b07347caa 100755 --- a/tests/test_optuna_run_config_generator.py +++ b/tests/test_optuna_run_config_generator.py @@ -93,7 +93,7 @@ def test_max_number_of_configs_to_search_count(self): """ Test count based max num of configs to search """ - config = self._create_config(additional_args=["--optuna_max_trials", "6"]) + config = self._create_config(additional_args=["--optuna-max-trials", "6"]) self._rcg._config = config @@ -109,9 +109,9 @@ def test_max_number_of_configs_to_search_both(self): """ config = self._create_config( additional_args=[ - "--optuna_max_trials", + "--optuna-max-trials", "6", - "--max_percentage_of_search_space", + "--max-percentage-of-search-space", "3", ] ) @@ -141,7 +141,7 @@ def test_min_number_of_configs_to_search_count(self): """ Test count based min num of configs to search """ - config = self._create_config(additional_args=["--optuna_min_trials", "12"]) + config = self._create_config(additional_args=["--optuna-min-trials", "12"]) self._rcg._config = config @@ -157,9 +157,9 @@ def test_min_number_of_configs_to_search_both(self): """ config = self._create_config( additional_args=[ - "--optuna_min_trials", + "--optuna-min-trials", "6", - "--min_percentage_of_search_space", + "--min-percentage-of-search-space", "3", ] ) From eb585542a5d0e445bc2e9763f511478d7780cee9 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Fri, 12 Jul 2024 15:36:32 +0000 Subject: [PATCH 06/10] Attempt at fixing BLS test --- qa/L0_optuna_bls_model/check_results.py | 8 ++++---- qa/L0_optuna_bls_model/test.sh | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py index 2a7577df2..07fa80f65 100755 --- a/qa/L0_optuna_bls_model/check_results.py +++ b/qa/L0_optuna_bls_model/check_results.py @@ -55,13 +55,13 @@ def check_profile_logs(self): # Composing model - add: # instance_group: 1 to 5 (5) # - # Minimum number of trials: 10 (40% of search space) - # Maximum number of trials: 20 (80% of search space) + # Minimum number of trials: 5 (20% of search space) + # Maximum number of trials: 10 (40% of search space) # # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default # 0 because all concurrencies could have been tested during the optuna run - expected_min_num_measurements = 10 + 0 - expected_max_num_measurements = 20 + 36 + expected_min_num_measurements = 5 + 0 + expected_max_num_measurements = 10 + 36 for model in self._models: token = f"Profiling {model}_config" diff --git a/qa/L0_optuna_bls_model/test.sh b/qa/L0_optuna_bls_model/test.sh index 542782d47..35e2d2331 100755 --- a/qa/L0_optuna_bls_model/test.sh +++ b/qa/L0_optuna_bls_model/test.sh @@ -53,9 +53,10 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost: MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" -MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --min-percentage-of-search-space 40 --max-percentage-of-search-space 80" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --min-percentage-of-search-space 20 --max-percentage-of-search-space 40" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-detailed-reports --triton-output-path=$SERVER_LOG" MODEL_ANALYZER_SUBCOMMAND="profile" +MODEL_ANALYZER_GLOBAL_OPTIONS="-v" run_analyzer From 1cc963de3c7a3366937c5757a5031541d5c25d4d Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Fri, 12 Jul 2024 21:31:06 +0000 Subject: [PATCH 07/10] Adding debug prints to all optuna tests --- qa/L0_optuna_bls_model/check_results.py | 11 ++++++----- qa/L0_optuna_bls_model/test.sh | 2 +- qa/L0_optuna_ensemble_model/test.sh | 1 + qa/L0_optuna_multi_model/test.sh | 2 ++ qa/L0_optuna_search/test.sh | 2 ++ 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py index 07fa80f65..998628dfe 100755 --- a/qa/L0_optuna_bls_model/check_results.py +++ b/qa/L0_optuna_bls_model/check_results.py @@ -49,19 +49,20 @@ def check_profile_logs(self): with open(self._analyzer_log, "r") as f: log_contents = f.read() - # Number of configs in search space: 25 + # Number of configs in search space: 275 # Model - bls: + # concurrency: 1 to 1024 (11) # instance_group: 1 to 5 (5) # Composing model - add: # instance_group: 1 to 5 (5) # - # Minimum number of trials: 5 (20% of search space) - # Maximum number of trials: 10 (40% of search space) + # Minimum number of trials: 14 (5% of search space) + # Maximum number of trials: 28 (10% of search space) # # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default # 0 because all concurrencies could have been tested during the optuna run - expected_min_num_measurements = 5 + 0 - expected_max_num_measurements = 10 + 36 + expected_min_num_measurements = 14 + 0 + expected_max_num_measurements = 28 + 36 for model in self._models: token = f"Profiling {model}_config" diff --git a/qa/L0_optuna_bls_model/test.sh b/qa/L0_optuna_bls_model/test.sh index 35e2d2331..aefb21d4a 100755 --- a/qa/L0_optuna_bls_model/test.sh +++ b/qa/L0_optuna_bls_model/test.sh @@ -53,7 +53,7 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --triton-metrics-url http://localhost: MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --output-model-repository-path $OUTPUT_MODEL_REPOSITORY --override-output-model-repository" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS -e $EXPORT_PATH --checkpoint-directory $CHECKPOINT_DIRECTORY --filename-server-only=$FILENAME_SERVER_ONLY" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_INFERENCE_MODEL --filename-model-gpu=$FILENAME_GPU_MODEL" -MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --min-percentage-of-search-space 20 --max-percentage-of-search-space 40" +MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-detailed-reports --triton-output-path=$SERVER_LOG" MODEL_ANALYZER_SUBCOMMAND="profile" MODEL_ANALYZER_GLOBAL_OPTIONS="-v" diff --git a/qa/L0_optuna_ensemble_model/test.sh b/qa/L0_optuna_ensemble_model/test.sh index ece65cd23..82db6c97b 100755 --- a/qa/L0_optuna_ensemble_model/test.sh +++ b/qa/L0_optuna_ensemble_model/test.sh @@ -55,6 +55,7 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_I MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-detailed-reports --triton-output-path=$SERVER_LOG" MODEL_ANALYZER_SUBCOMMAND="profile" +MODEL_ANALYZER_GLOBAL_OPTIONS="-v" run_analyzer diff --git a/qa/L0_optuna_multi_model/test.sh b/qa/L0_optuna_multi_model/test.sh index 909397984..18d41e0f6 100755 --- a/qa/L0_optuna_multi_model/test.sh +++ b/qa/L0_optuna_multi_model/test.sh @@ -53,6 +53,8 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --filename-model-inference=$FILENAME_I MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna --skip-summary-reports" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --min-percentage-of-search-space 2 --max-percentage-of-search-space 4" MODEL_ANALYZER_SUBCOMMAND="profile" +MODEL_ANALYZER_GLOBAL_OPTIONS="-v" + run_analyzer if [ $? -ne 0 ]; then echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" diff --git a/qa/L0_optuna_search/test.sh b/qa/L0_optuna_search/test.sh index be107521a..da74dee1a 100755 --- a/qa/L0_optuna_search/test.sh +++ b/qa/L0_optuna_search/test.sh @@ -54,6 +54,8 @@ MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --run-config-search-mode optuna" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --latency-budget 10" MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS --skip-summary-reports" MODEL_ANALYZER_SUBCOMMAND="profile" +MODEL_ANALYZER_GLOBAL_OPTIONS="-v" + run_analyzer if [ $? -ne 0 ]; then echo -e "\n***\n*** Test Failed. model-analyzer $MODEL_ANALYZER_SUBCOMMAND exited with non-zero exit code. \n***" From e8a7446ac0790d8d78ac370050e7071a472b3de8 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Mon, 15 Jul 2024 17:26:52 +0000 Subject: [PATCH 08/10] Fixing copyright dates --- qa/L0_ensemble_model/check_results.py | 4 ++-- qa/L0_optuna_bls_model/check_results.py | 5 ++--- qa/L0_optuna_bls_model/test.sh | 2 +- qa/L0_optuna_bls_model/test_config_generator.py | 5 ++--- qa/L0_optuna_ensemble_model/test.sh | 2 +- qa/L0_optuna_ensemble_model/test_config_generator.py | 5 ++--- qa/L0_optuna_multi_model/check_results.py | 5 ++--- qa/L0_optuna_multi_model/test.sh | 2 +- qa/L0_optuna_multi_model/test_config_generator.py | 5 ++--- qa/L0_optuna_search/check_results.py | 5 ++--- qa/L0_optuna_search/test_config_generator.py | 5 ++--- 11 files changed, 19 insertions(+), 26 deletions(-) diff --git a/qa/L0_ensemble_model/check_results.py b/qa/L0_ensemble_model/check_results.py index 649fde29f..599a716de 100755 --- a/qa/L0_ensemble_model/check_results.py +++ b/qa/L0_ensemble_model/check_results.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 - -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py index 998628dfe..ca71f87c9 100755 --- a/qa/L0_optuna_bls_model/check_results.py +++ b/qa/L0_optuna_bls_model/check_results.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_bls_model/test.sh b/qa/L0_optuna_bls_model/test.sh index aefb21d4a..67520b1f6 100755 --- a/qa/L0_optuna_bls_model/test.sh +++ b/qa/L0_optuna_bls_model/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_bls_model/test_config_generator.py b/qa/L0_optuna_bls_model/test_config_generator.py index 67ab0edbe..242a01fe5 100755 --- a/qa/L0_optuna_bls_model/test_config_generator.py +++ b/qa/L0_optuna_bls_model/test_config_generator.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_ensemble_model/test.sh b/qa/L0_optuna_ensemble_model/test.sh index 82db6c97b..4f231e6b8 100755 --- a/qa/L0_optuna_ensemble_model/test.sh +++ b/qa/L0_optuna_ensemble_model/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_ensemble_model/test_config_generator.py b/qa/L0_optuna_ensemble_model/test_config_generator.py index 15125d425..f5927f343 100755 --- a/qa/L0_optuna_ensemble_model/test_config_generator.py +++ b/qa/L0_optuna_ensemble_model/test_config_generator.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_multi_model/check_results.py b/qa/L0_optuna_multi_model/check_results.py index 79cfceed4..baec23d7f 100755 --- a/qa/L0_optuna_multi_model/check_results.py +++ b/qa/L0_optuna_multi_model/check_results.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_multi_model/test.sh b/qa/L0_optuna_multi_model/test.sh index 18d41e0f6..165353d0d 100755 --- a/qa/L0_optuna_multi_model/test.sh +++ b/qa/L0_optuna_multi_model/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_multi_model/test_config_generator.py b/qa/L0_optuna_multi_model/test_config_generator.py index f3bdb7a19..594032e15 100755 --- a/qa/L0_optuna_multi_model/test_config_generator.py +++ b/qa/L0_optuna_multi_model/test_config_generator.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_search/check_results.py b/qa/L0_optuna_search/check_results.py index 9e3bcaf0e..2aaa98894 100755 --- a/qa/L0_optuna_search/check_results.py +++ b/qa/L0_optuna_search/check_results.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qa/L0_optuna_search/test_config_generator.py b/qa/L0_optuna_search/test_config_generator.py index f3bdb7a19..594032e15 100755 --- a/qa/L0_optuna_search/test_config_generator.py +++ b/qa/L0_optuna_search/test_config_generator.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 299b24ce64b0caf5ecba475dfeb2593b99ef044b Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Mon, 15 Jul 2024 17:36:05 +0000 Subject: [PATCH 09/10] Fixing copyright dates --- qa/L0_optuna_bls_model/check_results.py | 5 +++-- qa/L0_optuna_ensemble_model/check_results.py | 7 +++++-- qa/L0_optuna_multi_model/check_results.py | 5 +++-- qa/L0_optuna_search/check_results.py | 6 ++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py index ca71f87c9..52d266679 100755 --- a/qa/L0_optuna_bls_model/check_results.py +++ b/qa/L0_optuna_bls_model/check_results.py @@ -58,8 +58,9 @@ def check_profile_logs(self): # Minimum number of trials: 14 (5% of search space) # Maximum number of trials: 28 (10% of search space) # - # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default - # 0 because all concurrencies could have been tested during the optuna run + # Then you have 4 configs (top-3 + default) x (0-9) concurrencies + # 0 because all concurrencies could have been profiled during the optuna run + # 9 because all but one of the concurrencies from 1..1024 still need profiling expected_min_num_measurements = 14 + 0 expected_max_num_measurements = 28 + 36 diff --git a/qa/L0_optuna_ensemble_model/check_results.py b/qa/L0_optuna_ensemble_model/check_results.py index 798afa70b..986668261 100755 --- a/qa/L0_optuna_ensemble_model/check_results.py +++ b/qa/L0_optuna_ensemble_model/check_results.py @@ -59,8 +59,11 @@ def check_profile_logs(self): # Minimum number of trials: 13 (5% of search space) # Maximum number of trials: 27 (10% of search space) # - # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default - # 0 because all concurrencies could have been tested during the optuna run + # + # Then you have 4 configs (top-3 + default) x (0-9) concurrencies + # 0 because all concurrencies could have been profiled during the optuna run + # 9 because all but one of the concurrencies from 1..1024 still need profiling + expected_min_num_measurements = 13 + 0 expected_max_num_measurements = 27 + 36 diff --git a/qa/L0_optuna_multi_model/check_results.py b/qa/L0_optuna_multi_model/check_results.py index baec23d7f..d89c3ef97 100755 --- a/qa/L0_optuna_multi_model/check_results.py +++ b/qa/L0_optuna_multi_model/check_results.py @@ -59,8 +59,9 @@ def check_profile_logs(self): # Minimum number of trials: 32 (2% of search space) # Maximum number of trials: 64 (4% of search space) # - # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default - # 0 because all concurrencies could have been tested during the optuna run + # Then you have 4 configs (top-3 + default) x (0-9) concurrencies + # 0 because all concurrencies could have been profiled during the optuna run + # 9 because all but one of the concurrencies from 1..1024 still need profiling expected_min_num_measurements = 32 + 0 expected_max_num_measurements = 64 + 36 diff --git a/qa/L0_optuna_search/check_results.py b/qa/L0_optuna_search/check_results.py index 2aaa98894..3f4bb0246 100755 --- a/qa/L0_optuna_search/check_results.py +++ b/qa/L0_optuna_search/check_results.py @@ -57,8 +57,10 @@ def check_profile_logs(self): # Minimum number of trials: 22 (5% of search space) # Maximum number of trials: 44 (10% of search space) # - # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default - # 0 because all concurrencies could have been tested during the optuna run + # Then you have 4 configs (top-3 + default) x (0-9) concurrencies + # 0 because all concurrencies could have been profiled during the optuna run + # 9 because all but one of the concurrencies from 1..1024 still need profiling + expected_min_num_measurements = 22 expected_max_num_measurements = 44 + 36 for model in self._models: From 3d1dc7527d7d8b181c417be3eb9220a1177fbdea Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Mon, 15 Jul 2024 17:36:49 +0000 Subject: [PATCH 10/10] Revert "Fixing copyright dates" This reverts commit 299b24ce64b0caf5ecba475dfeb2593b99ef044b. --- qa/L0_optuna_bls_model/check_results.py | 5 ++--- qa/L0_optuna_ensemble_model/check_results.py | 7 ++----- qa/L0_optuna_multi_model/check_results.py | 5 ++--- qa/L0_optuna_search/check_results.py | 6 ++---- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/qa/L0_optuna_bls_model/check_results.py b/qa/L0_optuna_bls_model/check_results.py index 52d266679..ca71f87c9 100755 --- a/qa/L0_optuna_bls_model/check_results.py +++ b/qa/L0_optuna_bls_model/check_results.py @@ -58,9 +58,8 @@ def check_profile_logs(self): # Minimum number of trials: 14 (5% of search space) # Maximum number of trials: 28 (10% of search space) # - # Then you have 4 configs (top-3 + default) x (0-9) concurrencies - # 0 because all concurrencies could have been profiled during the optuna run - # 9 because all but one of the concurrencies from 1..1024 still need profiling + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run expected_min_num_measurements = 14 + 0 expected_max_num_measurements = 28 + 36 diff --git a/qa/L0_optuna_ensemble_model/check_results.py b/qa/L0_optuna_ensemble_model/check_results.py index 986668261..798afa70b 100755 --- a/qa/L0_optuna_ensemble_model/check_results.py +++ b/qa/L0_optuna_ensemble_model/check_results.py @@ -59,11 +59,8 @@ def check_profile_logs(self): # Minimum number of trials: 13 (5% of search space) # Maximum number of trials: 27 (10% of search space) # - # - # Then you have 4 configs (top-3 + default) x (0-9) concurrencies - # 0 because all concurrencies could have been profiled during the optuna run - # 9 because all but one of the concurrencies from 1..1024 still need profiling - + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run expected_min_num_measurements = 13 + 0 expected_max_num_measurements = 27 + 36 diff --git a/qa/L0_optuna_multi_model/check_results.py b/qa/L0_optuna_multi_model/check_results.py index d89c3ef97..baec23d7f 100755 --- a/qa/L0_optuna_multi_model/check_results.py +++ b/qa/L0_optuna_multi_model/check_results.py @@ -59,9 +59,8 @@ def check_profile_logs(self): # Minimum number of trials: 32 (2% of search space) # Maximum number of trials: 64 (4% of search space) # - # Then you have 4 configs (top-3 + default) x (0-9) concurrencies - # 0 because all concurrencies could have been profiled during the optuna run - # 9 because all but one of the concurrencies from 1..1024 still need profiling + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run expected_min_num_measurements = 32 + 0 expected_max_num_measurements = 64 + 36 diff --git a/qa/L0_optuna_search/check_results.py b/qa/L0_optuna_search/check_results.py index 3f4bb0246..2aaa98894 100755 --- a/qa/L0_optuna_search/check_results.py +++ b/qa/L0_optuna_search/check_results.py @@ -57,10 +57,8 @@ def check_profile_logs(self): # Minimum number of trials: 22 (5% of search space) # Maximum number of trials: 44 (10% of search space) # - # Then you have 4 configs (top-3 + default) x (0-9) concurrencies - # 0 because all concurrencies could have been profiled during the optuna run - # 9 because all but one of the concurrencies from 1..1024 still need profiling - + # Then you have 4 x (0-9) for the concurrency sweep on Top 3 + default + # 0 because all concurrencies could have been tested during the optuna run expected_min_num_measurements = 22 expected_max_num_measurements = 44 + 36 for model in self._models: