diff --git a/.github/workflows/python-unit-test.yml b/.github/workflows/python-unit-test.yml index 1a1cc8428..e1bc40171 100644 --- a/.github/workflows/python-unit-test.yml +++ b/.github/workflows/python-unit-test.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 diff --git a/user_tools/pyproject.toml b/user_tools/pyproject.toml index 0601ab968..03225b914 100644 --- a/user_tools/pyproject.toml +++ b/user_tools/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "fastcore==1.5.29", "fire>=0.5.0", "pandas==1.4.3", - "pyYAML==6.0", + "pyYAML>=6.0", "tabulate==0.8.10", "importlib-resources==5.10.2", "requests==2.31.0", diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py index 1be2e92c0..76447bfb8 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py @@ -62,7 +62,8 @@ def _set_remaining_configuration_list(self) -> None: # Skip if the property already set continue prop_cmd = config_cmd_prefix[:] - prop_cmd.append(f'{prop_entry.get("section")}/{prop_entry_key}') + section_entry = prop_entry.get('section') + prop_cmd.append(f'{section_entry}/{prop_entry_key}') cmd_args = { 'cmd': prop_cmd, } diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/emr.py b/user_tools/src/spark_rapids_pytools/cloud_api/emr.py index 302d0b433..32e8314be 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/emr.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/emr.py @@ -138,9 +138,10 @@ def _list_inconsistent_configurations(self) -> list: if not (emr_pem_path.endswith('.pem') or emr_pem_path.endswith('ppk')): incorrect_envs.append(f'Private key file path [{emr_pem_path}] should be ppk or pem format') else: + tools_env_k = Utils.find_full_rapids_tools_env_key('KEY_PAIR_PATH') incorrect_envs.append( f'Private key file path is not set. It is required to SSH on driver node. ' - f'Set {Utils.find_full_rapids_tools_env_key("KEY_PAIR_PATH")}') + f'Set {tools_env_k}') return incorrect_envs def pull_cluster_props_by_args(self, args: dict) -> str: diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py index 1d801ea63..eb2626122 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py @@ -453,9 +453,11 @@ def append_to_cmd(original_cmd, extra_args: list) -> Any: for arg_entry in piggyback_args_raw: if is_sdk_cmd(cmd, arg_entry['sdkCommand']): # we should apply the - piggyback_args.append(f'--{arg_entry["argKey"]}') + arg_k = arg_entry['argKey'] + piggyback_args.append(f'--{arg_k}') if 'argValue' in arg_entry: - piggyback_args.append(f'{arg_entry["argValue"]}') + arg_v = arg_entry['argValue'] + piggyback_args.append(f'{arg_v}') else: arg_value = self.get_env_var(arg_entry['confProperty']) piggyback_args.append(arg_value) diff --git a/user_tools/src/spark_rapids_pytools/common/utilities.py b/user_tools/src/spark_rapids_pytools/common/utilities.py index 74c6b2e57..760e3281f 100644 --- a/user_tools/src/spark_rapids_pytools/common/utilities.py +++ b/user_tools/src/spark_rapids_pytools/common/utilities.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -328,8 +328,8 @@ def process_credentials_option(cmd: list): if len(std_error_lines) > 0: error_lines = Utils.gen_multiline_str(std_error_lines) stderr_str = f'\n{error_lines}' - processed_cmd_args = process_credentials_option(cmd_args) - cmd_err_msg = f'Error invoking CMD <{Utils.gen_joined_str(" ", processed_cmd_args)}>: {stderr_str}' + processed_cmd_args = Utils.gen_joined_str(' ', process_credentials_option(cmd_args)) + cmd_err_msg = f'Error invoking CMD <{processed_cmd_args}>: {stderr_str}' raise RuntimeError(f'{cmd_err_msg}') self.out_std = c.stdout if isinstance(c.stdout, str) else c.stdout.decode('utf-8', errors='ignore') diff --git a/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py b/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py index 3ef8f3604..511fdafc6 100644 --- a/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py +++ b/user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py @@ -364,13 +364,14 @@ def _calculate_spark_settings(self, worker_info: NodeHWInfo) -> dict: pinned_mem = min(constants.get('maxPinnedMemoryMB'), executor_container_mem - executor_heap - executor_mem_overhead - pageable_pool) executor_mem_overhead += pinned_mem + pageable_pool + max_sql_files_partitions = constants.get('maxSqlFilesPartitionsMB') res = { 'spark.executor.cores': num_executor_cores, 'spark.executor.memory': f'{executor_heap}m', 'spark.executor.memoryOverhead': f'{executor_mem_overhead}m', 'spark.rapids.sql.concurrentGpuTasks': gpu_concurrent_tasks, 'spark.rapids.memory.pinnedPool.size': f'{pinned_mem}m', - 'spark.sql.files.maxPartitionBytes': f'{constants.get("maxSqlFilesPartitionsMB")}m', + 'spark.sql.files.maxPartitionBytes': f'{max_sql_files_partitions}m', 'spark.task.resource.gpu.amount': 1 / num_executor_cores, 'spark.rapids.shuffle.multiThreaded.reader.threads': num_executor_cores, 'spark.rapids.shuffle.multiThreaded.writer.threads': num_executor_cores, @@ -394,7 +395,7 @@ def _process_jar_arg(self): self.ctxt.get_local_work_dir(), fail_ok=False, create_dir=True) - self.logger.info('RAPIDS accelerator jar is downloaded to work_dir %s', jar_path) + self.logger.info('RAPIDS accelerator tools jar is downloaded to work_dir %s', jar_path) # get the jar file name jar_file_name = FSUtil.get_resource_name(jar_path) version_match = re.search(r'\d{2}\.\d{2}\.\d+', jar_file_name) diff --git a/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py b/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py index a728bfafc..b9b847938 100644 --- a/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py +++ b/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py @@ -167,6 +167,7 @@ def detect_platform_from_eventlogs_prefix(self): ev_logs_path = CspPath(self.get_eventlogs().split(',')[0]) storage_type = ev_logs_path.get_storage_name() self.p_args['toolArgs']['platform'] = map_storage_to_platform[storage_type] + self.logger.info('Detected platform from eventlogs prefix: %s', self.p_args['toolArgs']['platform'].name) def validate_onprem_with_cluster_name(self): # this field has already been populated during initialization @@ -319,7 +320,7 @@ def define_invalid_arg_cases(self): [ArgValueCase.VALUE_A, ArgValueCase.VALUE_B, ArgValueCase.UNDEFINED] ] } - self.rejected['Invalid Jar Argument'] = { + self.rejected['Jar Argument'] = { 'valid': False, 'callable': partial(self.validate_jar_argument_is_valid), 'cases': [ @@ -443,9 +444,10 @@ def build_tools_args(self) -> dict: self.p_args['toolArgs']['targetPlatform'] = None else: if not self.p_args['toolArgs']['targetPlatform'] in equivalent_pricing_list: + target_platform = self.p_args['toolArgs']['targetPlatform'] raise PydanticCustomError( 'invalid_argument', - f'The platform [{self.p_args["toolArgs"]["targetPlatform"]}] is currently ' + f'The platform [{target_platform}] is currently ' f'not supported to calculate savings from [{runtime_platform}] cluster\n Error:') else: # target platform is not set, then we disable cost savings if the runtime platform if diff --git a/user_tools/tox.ini b/user_tools/tox.ini index e70d824f9..8bccbdf50 100644 --- a/user_tools/tox.ini +++ b/user_tools/tox.ini @@ -5,7 +5,7 @@ [tox] envlist = - python{3.8,3.9,3.10,3.11} + python{3.8,3.9,3.10,3.11,3.12} coverage pylint flake8 @@ -17,6 +17,7 @@ python = 3.9: python3.9, pylint, flake8 3.10: python3.10, pylint, flake8 3.11: python3.11, pylint, flake8 + 3.12: python3.12, pylint, flake8 [testenv] deps = @@ -45,7 +46,7 @@ commands = coverage combine coverage report depends = - python{3.8,3.9,3.10} + python{3.8,3.9,3.10,3.11,3.12} [coverage:paths] source = src/spark_rapids_pytools