From 7f02ab002c4bdcf59b9d2b7ad216562dc6e7ec4e Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Mon, 6 May 2024 19:27:05 -0700 Subject: [PATCH 1/6] Remove unnecessary wait in case of failed stub creation --- .../model_control/model_control_test.py | 18 ++++++ qa/L0_backend_python/model_control/test.sh | 3 + qa/python_models/faulty_model/1/model.py | 55 +++++++++++++++++++ qa/python_models/faulty_model/config.pbtxt | 55 +++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 qa/python_models/faulty_model/1/model.py create mode 100644 qa/python_models/faulty_model/config.pbtxt diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index 9ccb73df4f..d8e39c738b 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -82,6 +82,24 @@ def test_model_reload(self): self.assertFalse(client.is_model_ready(model_name)) self.assertFalse(client.is_model_ready(ensemble_model_name)) + def test_faulty_model_load(self): + working_model_name = "identity_fp32" + faulty_model_name = "faulty_model" + with httpclient.InferenceServerClient(f"{_tritonserver_ipaddr}:8000") as client: + for _ in range(5): + # Load a correct model + client.load_model(working_model_name) + # Load a faulty model + # Load a faulty model + try: + client.load_model(faulty_model_name) + except: + pass + # Check if server is responsive + self.assertTrue(client.is_model_ready(working_model_name)) + # Verify faulty model is not loaded + self.assertFalse(client.is_model_ready(faulty_model_name)) + if __name__ == "__main__": unittest.main() diff --git a/qa/L0_backend_python/model_control/test.sh b/qa/L0_backend_python/model_control/test.sh index e2c22f2685..a296d1c4af 100755 --- a/qa/L0_backend_python/model_control/test.sh +++ b/qa/L0_backend_python/model_control/test.sh @@ -37,9 +37,12 @@ source ../../common/util.sh mkdir -p models/identity_fp32/1/ mkdir -p models/simple_identity_fp32/1/ +mkdir -p models/faulty_model/ + cp ../../python_models/identity_fp32/model.py ./models/identity_fp32/1/model.py cp ../../python_models/identity_fp32/config.pbtxt ./models/identity_fp32/config.pbtxt cp ../../python_models/simple_identity_fp32/config.pbtxt ./models/simple_identity_fp32/config.pbtxt +cp -r ../../python_models/faulty_model/. ./models/faulty_model run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/python_models/faulty_model/1/model.py b/qa/python_models/faulty_model/1/model.py new file mode 100644 index 0000000000..337d99d147 --- /dev/null +++ b/qa/python_models/faulty_model/1/model.py @@ -0,0 +1,55 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import time + +import numpy as np +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + @staticmethod + def auto_complete_config(auto_complete_model_config): + # TYPO to cause error in load + input0 = {"nameas": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]} + auto_complete_model_config.add_input(input0) + + def initialize(self, args): + self.model_config = json.loads(args["model_config"]) + + def execute(self, requests): + """This function is called on inference request.""" + # Less than collector timeout which is 10 + time.sleep(2) + responses = [] + for _ in requests: + # Include one of each specially parsed JSON value: nan, inf, and -inf + out_0 = np.array([1], dtype=np.float32) + out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0) + responses.append(pb_utils.InferenceResponse([out_tensor_0])) + + return responses diff --git a/qa/python_models/faulty_model/config.pbtxt b/qa/python_models/faulty_model/config.pbtxt new file mode 100644 index 0000000000..73ac8b2fb7 --- /dev/null +++ b/qa/python_models/faulty_model/config.pbtxt @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +name: "faulty_model" +backend: "python" +input [ + { + name: "INPUT0" + data_type: TYPE_FP32 + dims: [ -1 ] + }, + { + name: "INPUT1" + data_type: TYPE_FP32 + dims: [ -1 ] + }, + { + name: "INPUT2" + data_type: TYPE_FP32 + dims: [ -1 ] + } +] + +output [ + { + name: "OUTPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] + +instance_group [{ kind: KIND_CPU }] \ No newline at end of file From b5312efd336fd2288e4e024ffd51f888bf116eec Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Tue, 7 May 2024 12:59:13 -0700 Subject: [PATCH 2/6] Pre-Commit fix --- qa/python_models/faulty_model/1/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/python_models/faulty_model/1/model.py b/qa/python_models/faulty_model/1/model.py index 337d99d147..e27631c0a7 100644 --- a/qa/python_models/faulty_model/1/model.py +++ b/qa/python_models/faulty_model/1/model.py @@ -37,7 +37,7 @@ def auto_complete_config(auto_complete_model_config): # TYPO to cause error in load input0 = {"nameas": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]} auto_complete_model_config.add_input(input0) - + def initialize(self, args): self.model_config = json.loads(args["model_config"]) From 1cad688da40bfe8756ff63183e7506f33d8e813b Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 8 May 2024 10:50:37 -0700 Subject: [PATCH 3/6] Fix pipeline --- .../model_control/model_control_test.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index d8e39c738b..ef339fa7c3 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -86,20 +86,19 @@ def test_faulty_model_load(self): working_model_name = "identity_fp32" faulty_model_name = "faulty_model" with httpclient.InferenceServerClient(f"{_tritonserver_ipaddr}:8000") as client: - for _ in range(5): - # Load a correct model - client.load_model(working_model_name) - # Load a faulty model - # Load a faulty model - try: - client.load_model(faulty_model_name) - except: - pass - # Check if server is responsive - self.assertTrue(client.is_model_ready(working_model_name)) - # Verify faulty model is not loaded - self.assertFalse(client.is_model_ready(faulty_model_name)) - + # Load a correct model + client.load_model(working_model_name) + # Load a faulty model + # Load a faulty model + try: + client.load_model(faulty_model_name) + except: + pass + # Check if server is responsive + self.assertTrue(client.is_model_ready(working_model_name)) + # Verify faulty model is not loaded + self.assertFalse(client.is_model_ready(faulty_model_name)) + client.unload_model(working_model_name) if __name__ == "__main__": unittest.main() From 48d431e61220dd07d55f8df798dc019734f30ca2 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 8 May 2024 10:51:00 -0700 Subject: [PATCH 4/6] Pre-commit --- qa/L0_backend_python/model_control/model_control_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index ef339fa7c3..84fa660383 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -100,5 +100,6 @@ def test_faulty_model_load(self): self.assertFalse(client.is_model_ready(faulty_model_name)) client.unload_model(working_model_name) + if __name__ == "__main__": unittest.main() From 4aee04642be781aa985a0523ab11442f5c78f32e Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 8 May 2024 12:32:49 -0700 Subject: [PATCH 5/6] Review Comments fixed --- .../model_control/model_control_test.py | 11 ++-- qa/L0_backend_python/model_control/test.sh | 4 +- qa/python_models/faulty_model/1/model.py | 55 ------------------- qa/python_models/faulty_model/config.pbtxt | 55 ------------------- 4 files changed, 7 insertions(+), 118 deletions(-) delete mode 100644 qa/python_models/faulty_model/1/model.py delete mode 100644 qa/python_models/faulty_model/config.pbtxt diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index 84fa660383..b9770fe02f 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -84,16 +84,15 @@ def test_model_reload(self): def test_faulty_model_load(self): working_model_name = "identity_fp32" - faulty_model_name = "faulty_model" + faulty_model_name = "auto_complete_error" with httpclient.InferenceServerClient(f"{_tritonserver_ipaddr}:8000") as client: # Load a correct model client.load_model(working_model_name) # Load a faulty model - # Load a faulty model - try: - client.load_model(faulty_model_name) - except: - pass + with self.assertRaises(InferenceServerException) as cm: + _ = client.load_model(faulty_model_name) + self.assertIn("load failed for model", str(cm.exception)) + # Check if server is responsive self.assertTrue(client.is_model_ready(working_model_name)) # Verify faulty model is not loaded diff --git a/qa/L0_backend_python/model_control/test.sh b/qa/L0_backend_python/model_control/test.sh index a296d1c4af..3ca680cd8e 100755 --- a/qa/L0_backend_python/model_control/test.sh +++ b/qa/L0_backend_python/model_control/test.sh @@ -37,12 +37,12 @@ source ../../common/util.sh mkdir -p models/identity_fp32/1/ mkdir -p models/simple_identity_fp32/1/ -mkdir -p models/faulty_model/ +mkdir -p models/auto_complete_error/1/ +cp ../../python_models/auto_complete_error/model.py ./models/auto_complete_error/1/ cp ../../python_models/identity_fp32/model.py ./models/identity_fp32/1/model.py cp ../../python_models/identity_fp32/config.pbtxt ./models/identity_fp32/config.pbtxt cp ../../python_models/simple_identity_fp32/config.pbtxt ./models/simple_identity_fp32/config.pbtxt -cp -r ../../python_models/faulty_model/. ./models/faulty_model run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/python_models/faulty_model/1/model.py b/qa/python_models/faulty_model/1/model.py deleted file mode 100644 index e27631c0a7..0000000000 --- a/qa/python_models/faulty_model/1/model.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import time - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - @staticmethod - def auto_complete_config(auto_complete_model_config): - # TYPO to cause error in load - input0 = {"nameas": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]} - auto_complete_model_config.add_input(input0) - - def initialize(self, args): - self.model_config = json.loads(args["model_config"]) - - def execute(self, requests): - """This function is called on inference request.""" - # Less than collector timeout which is 10 - time.sleep(2) - responses = [] - for _ in requests: - # Include one of each specially parsed JSON value: nan, inf, and -inf - out_0 = np.array([1], dtype=np.float32) - out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0) - responses.append(pb_utils.InferenceResponse([out_tensor_0])) - - return responses diff --git a/qa/python_models/faulty_model/config.pbtxt b/qa/python_models/faulty_model/config.pbtxt deleted file mode 100644 index 73ac8b2fb7..0000000000 --- a/qa/python_models/faulty_model/config.pbtxt +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "faulty_model" -backend: "python" -input [ - { - name: "INPUT0" - data_type: TYPE_FP32 - dims: [ -1 ] - }, - { - name: "INPUT1" - data_type: TYPE_FP32 - dims: [ -1 ] - }, - { - name: "INPUT2" - data_type: TYPE_FP32 - dims: [ -1 ] - } -] - -output [ - { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] - -instance_group [{ kind: KIND_CPU }] \ No newline at end of file From de7fe5508bead7d1a1b71c67458607992dde01cf Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 8 May 2024 15:57:40 -0700 Subject: [PATCH 6/6] Pipeline crash fix --- qa/L0_backend_python/model_control/model_control_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index b9770fe02f..c984710f0b 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -97,7 +97,7 @@ def test_faulty_model_load(self): self.assertTrue(client.is_model_ready(working_model_name)) # Verify faulty model is not loaded self.assertFalse(client.is_model_ready(faulty_model_name)) - client.unload_model(working_model_name) + client.unload_model(working_model_name) if __name__ == "__main__":