Skip to content

Commit

Permalink
Merge pull request #106 from altheaden/modularization-and-cleanup
Browse files Browse the repository at this point in the history
Modularization and cleanup
  • Loading branch information
xylar authored Aug 7, 2023
2 parents 32ebdff + 3f147c7 commit a27d7f2
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 39 deletions.
2 changes: 2 additions & 0 deletions docs/developers_guide/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ seaice/api
unpickle_suite
setup_config
load_dependencies
pickle_step_after_run
serial.run_tests
serial.run_single_step
Expand Down
50 changes: 50 additions & 0 deletions polaris/run/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,53 @@ def setup_config(config_filename):
config = PolarisConfigParser()
config.add_from_file(config_filename)
return config


def load_dependencies(test_case, step):
"""
Load each dependency from its pickle file to pick up changes that may have
happened since it ran
Parameters
----------
test_case : polaris.testcase.TestCase
The test case object
step : polaris.step.Step
The step object
"""
for name, old_dependency in step.dependencies.items():
if old_dependency.cached:
continue

pickle_filename = os.path.join(old_dependency.work_dir,
'step_after_run.pickle')
if not os.path.exists(pickle_filename):
raise ValueError(f'The dependency {name} of '
f'{test_case.path} step {step.name} was '
f'not run.')

with open(pickle_filename, 'rb') as handle:
_, dependency = pickle.load(handle)
step.dependencies[name] = dependency


def pickle_step_after_run(test_case, step):
"""
Pickle a step after it has run so its dependencies will pick up the
changes
Parameters
----------
test_case : polaris.testcase.TestCase
The test case object
step : polaris.step.Step
The step object
"""
if step.is_dependency:
# pickle the test case and step for use at runtime
pickle_filename = os.path.join(step.work_dir, 'step_after_run.pickle')
with open(pickle_filename, 'wb') as handle:
pickle.dump((test_case, step), handle,
protocol=pickle.HIGHEST_PROTOCOL)
54 changes: 15 additions & 39 deletions polaris/run/serial.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@
run_command,
set_cores_per_node,
)
from polaris.run import setup_config, unpickle_suite
from polaris.run import (
load_dependencies,
pickle_step_after_run,
setup_config,
unpickle_suite,
)


def run_tests(suite_name, quiet=False, is_test_case=False, steps_to_run=None,
Expand Down Expand Up @@ -113,6 +118,7 @@ def run_single_step(step_is_subprocess=False):
test_case.steps_to_run = [step.name]
test_case.new_step_log_file = False

# This prevents infinite loop of subprocesses
if step_is_subprocess:
step.run_as_subprocess = False

Expand Down Expand Up @@ -164,6 +170,7 @@ def main():
help="Used internally by polaris to indicate that "
"a step is being run as a subprocess.")
args = parser.parse_args(sys.argv[2:])

if args.suite is not None:
# Running a specified suite from the base work directory
run_tests(args.suite, quiet=args.quiet)
Expand Down Expand Up @@ -433,7 +440,7 @@ def _run_step(test_case, step, new_log_file, available_resources,
f'{step.component.name}/{step.test_group.name}/'
f'{step.test_case.subdir}: {missing_files}')

_load_dependencies(test_case, step)
load_dependencies(test_case, step)

# each logger needs a unique name
logger_name = step.path.replace('/', '_')
Expand Down Expand Up @@ -483,7 +490,7 @@ def _run_step(test_case, step, new_log_file, available_resources,
step_logger.info('')
step.run()

_pickle_step_after_run(test_case, step)
pickle_step_after_run(test_case, step)

missing_files = list()
for output_file in step.outputs:
Expand All @@ -508,50 +515,19 @@ def _run_step_as_subprocess(test_case, step, new_log_file):
"""
logger = test_case.logger
cwd = os.getcwd()
test_name = step.path.replace('/', '_')
logger_name = step.path.replace('/', '_')
if new_log_file:
log_filename = f'{cwd}/{step.name}.log'
step.log_filename = log_filename
step_logger = None
else:
step_logger = logger
log_filename = None
with LoggingContext(name=test_name, logger=step_logger,

step.log_filename = log_filename

with LoggingContext(name=logger_name, logger=step_logger,
log_filename=log_filename) as step_logger:

os.chdir(step.work_dir)
step_args = ['polaris', 'serial', '--step_is_subprocess']
check_call(step_args, step_logger)


def _load_dependencies(test_case, step):
"""
Load each dependency from its pickle file to pick up changes that may have
happened since it ran
"""
for name, old_dependency in step.dependencies.items():
if old_dependency.cached:
continue

pickle_filename = os.path.join(old_dependency.work_dir,
'step_after_run.pickle')
if not os.path.exists(pickle_filename):
raise ValueError(f'The dependency {name} of '
f'{test_case.path} step {step.name} was '
f'not run.')

with open(pickle_filename, 'rb') as handle:
dep_test_case, dependency = pickle.load(handle)
step.dependencies[name] = dependency


def _pickle_step_after_run(test_case, step):
"""
Pickle a step after running so its dependencies will pick up the changes.
"""
if step.is_dependency:
# pickle the test case and step for use at runtime
pickle_filename = os.path.join(step.work_dir, 'step_after_run.pickle')
with open(pickle_filename, 'wb') as handle:
pickle.dump((test_case, step), handle,
protocol=pickle.HIGHEST_PROTOCOL)

0 comments on commit a27d7f2

Please sign in to comment.