Merge dev into main

Signed-off-by: spark-rapids automation <[email protected]>
NVIDIA · Jun 11, 2024 · a3e4988 · a3e4988
2 parents 5e01ac6 + 429ccec
commit a3e4988
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 7 deletions.
diff --git a/.github/workflows/python-unit-test.yml b/.github/workflows/python-unit-test.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
       - uses: actions/checkout@v4

diff --git a/user_tools/pyproject.toml b/user_tools/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "tabulate==0.8.10",
     "importlib-resources==5.10.2",
     "requests==2.31.0",
-    "packaging==23.0",
+    "packaging>=23.0",
     "certifi==2023.7.22",
     "idna==3.4",
     "urllib3==1.26.14",

diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py
@@ -905,7 +905,7 @@ def get_matching_executor_instance(self, cores_per_executor):
     def generate_cluster_configuration(self, render_args: dict):
         if not self.cluster_inference_supported:
             return None
-        template_path = Utils.resource_path(f'templates/cluster_template/{self.type_id}.ms')
+        template_path = Utils.resource_path(f'templates/cluster_template/{CspEnv.pretty_print(self.type_id)}.ms')
         return TemplateGenerator.render_template_file(template_path, render_args)
 
 

diff --git a/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py b/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py
@@ -95,8 +95,8 @@ def qualification(self,
                 stage duration less than 25% of app duration and speedups greater than 1.3x.
         :param estimation_model: Model used to calculate the estimated GPU duration and cost savings.
                It accepts one of the following:
-               "xgboost": an XGBoost model for GPU duration estimation
-               "speedups": set by default. It uses a simple static estimated speedup per operator.
+               "xgboost": An XGBoost model for GPU duration estimation. Set by default
+               "speedups": It uses a simple static estimated speedup per operator.
         :param cpu_cluster_price: the CPU cluster hourly price provided by the user.
         :param estimated_gpu_cluster_price: the GPU cluster hourly price provided by the user.
         :param cpu_discount: A percent discount for the cpu cluster cost in the form of an integer value

diff --git a/user_tools/src/spark_rapids_tools/enums.py b/user_tools/src/spark_rapids_tools/enums.py
@@ -167,4 +167,4 @@ class QualEstimationModel(EnumeratedType):
 
     @classmethod
     def get_default(cls):
-        return cls.SPEEDUPS
+        return cls.XGBOOST
diff --git a/user_tools/src/spark_rapids_tools/tools/qualx/model.py b/user_tools/src/spark_rapids_tools/tools/qualx/model.py
@@ -71,6 +71,16 @@ def train(
         cpu_aug_tbl = cpu_aug_tbl.copy()
         cpu_aug_tbl[label_col] = np.log(cpu_aug_tbl[label_col])
 
+    # remove nan label entries
+    original_num_rows = cpu_aug_tbl.shape[0]
+    cpu_aug_tbl = cpu_aug_tbl.loc[~cpu_aug_tbl[label_col].isna()].reset_index(
+        drop=True
+    )
+    if cpu_aug_tbl.shape[0] < original_num_rows:
+        logger.warn(
+            f'Removed {original_num_rows - cpu_aug_tbl.shape[0]} rows with NaN label values'
+        )
+
     # split into train/val/test sets
     X_train = cpu_aug_tbl.loc[cpu_aug_tbl['split'] == 'train', feature_cols]
     y_train = cpu_aug_tbl.loc[cpu_aug_tbl['split'] == 'train', label_col]

diff --git a/user_tools/tox.ini b/user_tools/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 envlist =
-    python{3.8,3.9,3.10}
+    python{3.8,3.9,3.10,3.11}
     coverage
     pylint
     flake8
@@ -16,6 +16,7 @@ python =
     3.8: python3.8, pylint, flake8
     3.9: python3.9, pylint, flake8
     3.10: python3.10, pylint, flake8
+    3.11: python3.11, pylint, flake8
 
 [testenv]
 deps =