Merge dev into main

Signed-off-by: spark-rapids automation <[email protected]>
NVIDIA · Jun 12, 2024 · 5bd805d · 5bd805d
2 parents a3e4988 + aaa0239
commit 5bd805d
Show file tree

Hide file tree

Showing 9 changed files with 23 additions and 15 deletions.
diff --git a/.github/workflows/python-unit-test.yml b/.github/workflows/python-unit-test.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
       - uses: actions/checkout@v4

diff --git a/user_tools/pyproject.toml b/user_tools/pyproject.toml
@@ -21,7 +21,7 @@ dependencies = [
     "fastcore==1.5.29",
     "fire>=0.5.0",
     "pandas==1.4.3",
-    "pyYAML==6.0",
+    "pyYAML>=6.0",
     "tabulate==0.8.10",
     "importlib-resources==5.10.2",
     "requests==2.31.0",

diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py
@@ -62,7 +62,8 @@ def _set_remaining_configuration_list(self) -> None:
                     # Skip if the property already set
                     continue
                 prop_cmd = config_cmd_prefix[:]
-                prop_cmd.append(f'{prop_entry.get("section")}/{prop_entry_key}')
+                section_entry = prop_entry.get('section')
+                prop_cmd.append(f'{section_entry}/{prop_entry_key}')
                 cmd_args = {
                     'cmd': prop_cmd,
                 }

diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/emr.py b/user_tools/src/spark_rapids_pytools/cloud_api/emr.py
@@ -138,9 +138,10 @@ def _list_inconsistent_configurations(self) -> list:
                 if not (emr_pem_path.endswith('.pem') or emr_pem_path.endswith('ppk')):
                     incorrect_envs.append(f'Private key file path [{emr_pem_path}] should be ppk or pem format')
         else:
+            tools_env_k = Utils.find_full_rapids_tools_env_key('KEY_PAIR_PATH')
             incorrect_envs.append(
                 f'Private key file path is not set. It is required to SSH on driver node. '
-                f'Set {Utils.find_full_rapids_tools_env_key("KEY_PAIR_PATH")}')
+                f'Set {tools_env_k}')
         return incorrect_envs
 
     def pull_cluster_props_by_args(self, args: dict) -> str:

diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py
@@ -453,9 +453,11 @@ def append_to_cmd(original_cmd, extra_args: list) -> Any:
         for arg_entry in piggyback_args_raw:
             if is_sdk_cmd(cmd, arg_entry['sdkCommand']):
                 # we should apply the
-                piggyback_args.append(f'--{arg_entry["argKey"]}')
+                arg_k = arg_entry['argKey']
+                piggyback_args.append(f'--{arg_k}')
                 if 'argValue' in arg_entry:
-                    piggyback_args.append(f'{arg_entry["argValue"]}')
+                    arg_v = arg_entry['argValue']
+                    piggyback_args.append(f'{arg_v}')
                 else:
                     arg_value = self.get_env_var(arg_entry['confProperty'])
                     piggyback_args.append(arg_value)

diff --git a/user_tools/src/spark_rapids_pytools/common/utilities.py b/user_tools/src/spark_rapids_pytools/common/utilities.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -328,8 +328,8 @@ def process_credentials_option(cmd: list):
             if len(std_error_lines) > 0:
                 error_lines = Utils.gen_multiline_str(std_error_lines)
                 stderr_str = f'\n{error_lines}'
-            processed_cmd_args = process_credentials_option(cmd_args)
-            cmd_err_msg = f'Error invoking CMD <{Utils.gen_joined_str(" ", processed_cmd_args)}>: {stderr_str}'
+            processed_cmd_args = Utils.gen_joined_str(' ', process_credentials_option(cmd_args))
+            cmd_err_msg = f'Error invoking CMD <{processed_cmd_args}>: {stderr_str}'
             raise RuntimeError(f'{cmd_err_msg}')
 
         self.out_std = c.stdout if isinstance(c.stdout, str) else c.stdout.decode('utf-8', errors='ignore')

diff --git a/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py b/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py
@@ -364,13 +364,14 @@ def _calculate_spark_settings(self, worker_info: NodeHWInfo) -> dict:
         pinned_mem = min(constants.get('maxPinnedMemoryMB'),
                          executor_container_mem - executor_heap - executor_mem_overhead - pageable_pool)
         executor_mem_overhead += pinned_mem + pageable_pool
+        max_sql_files_partitions = constants.get('maxSqlFilesPartitionsMB')
         res = {
             'spark.executor.cores': num_executor_cores,
             'spark.executor.memory': f'{executor_heap}m',
             'spark.executor.memoryOverhead': f'{executor_mem_overhead}m',
             'spark.rapids.sql.concurrentGpuTasks': gpu_concurrent_tasks,
             'spark.rapids.memory.pinnedPool.size': f'{pinned_mem}m',
-            'spark.sql.files.maxPartitionBytes': f'{constants.get("maxSqlFilesPartitionsMB")}m',
+            'spark.sql.files.maxPartitionBytes': f'{max_sql_files_partitions}m',
             'spark.task.resource.gpu.amount': 1 / num_executor_cores,
             'spark.rapids.shuffle.multiThreaded.reader.threads': num_executor_cores,
             'spark.rapids.shuffle.multiThreaded.writer.threads': num_executor_cores,
@@ -394,7 +395,7 @@ def _process_jar_arg(self):
                                                                 self.ctxt.get_local_work_dir(),
                                                                 fail_ok=False,
                                                                 create_dir=True)
-        self.logger.info('RAPIDS accelerator jar is downloaded to work_dir %s', jar_path)
+        self.logger.info('RAPIDS accelerator tools jar is downloaded to work_dir %s', jar_path)
         # get the jar file name
         jar_file_name = FSUtil.get_resource_name(jar_path)
         version_match = re.search(r'\d{2}\.\d{2}\.\d+', jar_file_name)

diff --git a/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py b/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py
@@ -167,6 +167,7 @@ def detect_platform_from_eventlogs_prefix(self):
         ev_logs_path = CspPath(self.get_eventlogs().split(',')[0])
         storage_type = ev_logs_path.get_storage_name()
         self.p_args['toolArgs']['platform'] = map_storage_to_platform[storage_type]
+        self.logger.info('Detected platform from eventlogs prefix: %s', self.p_args['toolArgs']['platform'].name)
 
     def validate_onprem_with_cluster_name(self):
         # this field has already been populated during initialization
@@ -319,7 +320,7 @@ def define_invalid_arg_cases(self):
                 [ArgValueCase.VALUE_A, ArgValueCase.VALUE_B, ArgValueCase.UNDEFINED]
             ]
         }
-        self.rejected['Invalid Jar Argument'] = {
+        self.rejected['Jar Argument'] = {
             'valid': False,
             'callable': partial(self.validate_jar_argument_is_valid),
             'cases': [
@@ -443,9 +444,10 @@ def build_tools_args(self) -> dict:
                 self.p_args['toolArgs']['targetPlatform'] = None
             else:
                 if not self.p_args['toolArgs']['targetPlatform'] in equivalent_pricing_list:
+                    target_platform = self.p_args['toolArgs']['targetPlatform']
                     raise PydanticCustomError(
                         'invalid_argument',
-                        f'The platform [{self.p_args["toolArgs"]["targetPlatform"]}] is currently '
+                        f'The platform [{target_platform}] is currently '
                         f'not supported to calculate savings from [{runtime_platform}] cluster\n  Error:')
         else:
             # target platform is not set, then we disable cost savings if the runtime platform if

diff --git a/user_tools/tox.ini b/user_tools/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 envlist =
-    python{3.8,3.9,3.10,3.11}
+    python{3.8,3.9,3.10,3.11,3.12}
     coverage
     pylint
     flake8
@@ -17,6 +17,7 @@ python =
     3.9: python3.9, pylint, flake8
     3.10: python3.10, pylint, flake8
     3.11: python3.11, pylint, flake8
+    3.12: python3.12, pylint, flake8
 
 [testenv]
 deps =
@@ -45,7 +46,7 @@ commands =
     coverage combine
     coverage report
 depends =
-    python{3.8,3.9,3.10}
+    python{3.8,3.9,3.10,3.11,3.12}
 
 [coverage:paths]
 source = src/spark_rapids_pytools