diff --git a/.github/workflows/run_dev_install_all_tests.yml b/.github/workflows/run_dev_install_all_tests.yml index 1ef7b3cc4..499fa19ce 100644 --- a/.github/workflows/run_dev_install_all_tests.yml +++ b/.github/workflows/run_dev_install_all_tests.yml @@ -93,16 +93,16 @@ jobs: with: name: idmtools_slurm_utils_test_results path: idmtools_slurm_utils/tests/reports/ - - name: run idmtools_platform_file tests + - name: run idmtools_platform_general tests run: | - cd idmtools_platform_file + cd idmtools_platform_general make test-all - - name: Upload idmtools_platform_file all test results + - name: Upload idmtools_platform_general all test results uses: actions/upload-artifact@v2 if: failure() with: - name: idmtools_platform_file_test_results - path: idmtools_platform_file/tests/reports/ + name: idmtools_platform_general_test_results + path: idmtools_platform_general/tests/reports/ - name: run idmtools_platform_comps tests run: | cd idmtools_platform_comps diff --git a/.github/workflows/run_pip_prod_all_tests.yml b/.github/workflows/run_pip_prod_all_tests.yml index 2d98ae1cc..1ebf5c9e8 100644 --- a/.github/workflows/run_pip_prod_all_tests.yml +++ b/.github/workflows/run_pip_prod_all_tests.yml @@ -89,6 +89,16 @@ jobs: with: name: idmtools_slurm_utils_test_results path: idmtools_slurm_utils/tests/reports/ + - name: run idmtools_platform_general tests + run: | + cd idmtools_platform_general + make test-all + - name: Upload idmtools_platform_general all test results + uses: actions/upload-artifact@v2 + if: failure() + with: + name: idmtools_platform_general_test_results + path: idmtools_platform_general/tests/reports/ - name: run idmtools_platform_comps tests run: | cd idmtools_platform_comps diff --git a/.github/workflows/run_pip_stage_all_tests.yml b/.github/workflows/run_pip_stage_all_tests.yml index f583b18cb..2d11491ed 100644 --- a/.github/workflows/run_pip_stage_all_tests.yml +++ b/.github/workflows/run_pip_stage_all_tests.yml @@ -93,6 +93,16 @@ jobs: with: name: idmtools_slurm_utils_test_results path: idmtools_slurm_utils/tests/reports/ + - name: run idmtools_platform_general tests + run: | + cd idmtools_platform_general + make test-all + - name: Upload idmtools_platform_general all test results + uses: actions/upload-artifact@v2 + if: failure() + with: + name: idmtools_platform_general_test_results + path: idmtools_platform_general/tests/reports/ - name: run idmtools_platform_comps tests run: | cd idmtools_platform_comps @@ -104,13 +114,3 @@ jobs: name: idmtools_platform_comps_test_results path: idmtools_platform_comps/tests/reports/ - - name: run idmtools_platform_file tests - run: | - cd idmtools_platform_file - make test-all - - name: Upload idmtools_platform_file all test results - uses: actions/upload-artifact@v2 - if: failure() - with: - name: idmtools_platform_file_test_results - path: idmtools_platform_file/tests/reports/ diff --git a/dev_scripts/bootstrap.py b/dev_scripts/bootstrap.py index 1ac348a72..b33272d36 100755 --- a/dev_scripts/bootstrap.py +++ b/dev_scripts/bootstrap.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -"""This scripts aids in setup of development environments. +""" +This scripts aids in setup of development environments. The script installs all the local packages defined by packages using development installs. @@ -23,12 +24,10 @@ from os.path import abspath, join, dirname from typing import List, Generator - -# on windows virtual env is not populated through pymake +# on windowns virtual env is not populated through pymake if sys.platform == "win32" and 'VIRTUAL_ENV' in os.environ: sys.path.insert(0, os.environ['VIRTUAL_ENV'] + "\\Lib\\site-packages") - script_dir = abspath(dirname(__file__)) base_directory = abspath(join(dirname(__file__), '..')) @@ -46,7 +45,7 @@ idmtools_platform_comps=data_class_default, idmtools_models=data_class_default, idmtools_platform_slurm=data_class_default, - idmtools_platform_file=data_class_default, + idmtools_platform_general=data_class_default, idmtools_slurm_utils=[], idmtools_test=[] ) @@ -54,7 +53,8 @@ def execute(cmd: List['str'], cwd: str = base_directory, ignore_error: bool = False) -> Generator[str, None, None]: - """Runs a command and filters output. + """ + Runs a command and filters output. Args: cmd: Command to run @@ -77,7 +77,8 @@ def execute(cmd: List['str'], cwd: str = base_directory, ignore_error: bool = Fa def process_output(output_line: str): - """Process output line for display. + """ + Process output line for display. This function adds coloring, filters output, and strips non-ascii characters(Docker builds have some odd characters) @@ -85,7 +86,7 @@ def process_output(output_line: str): output_line: Output line Returns: - None. Instead prints to log level on screen + None. Instead of prints to log level on screen """ # catch errors where possible if "FAILED [" in output_line: @@ -100,10 +101,11 @@ def process_output(output_line: str): def install_dev_packages(pip_url): - """Install the development packages. + """ + Install the development packages. This loops through all our idmtools packages and runs pip install -e . on each package - It also runs a pip install -r requirements from the docs directory. + It also runs a pip install -r requirements from the doc directory. Args: pip_url: Url to install package from @@ -116,18 +118,21 @@ def install_dev_packages(pip_url): extras_str = f"[{','.join(extras)}]" if extras else '' logger.info(f'Installing {package} with extras: {extras_str if extras_str else "None"} from {base_directory}') try: - for line in execute(["pip3", "install", "-e", f".{extras_str}", f"--extra-index-url={pip_url}"], cwd=join(base_directory, package)): + for line in execute(["pip3", "install", "-e", f".{extras_str}", f"--extra-index-url={pip_url}"], + cwd=join(base_directory, package)): process_output(line) except subprocess.CalledProcessError as e: logger.critical(f'{package} installed failed using {e.cmd} did not succeed') result = e.returncode logger.debug(f'Return Code: {result}') - for line in execute(["pip3", "install", "-r", "requirements.txt", f"--extra-index-url={pip_url}"], cwd=join(base_directory, 'docs')): + for line in execute(["pip3", "install", "-r", "requirements.txt", f"--extra-index-url={pip_url}"], + cwd=join(base_directory, 'docs')): process_output(line) def install_base_environment(pip_url): - """Installs the base packages needed for development environments. + """ + Installs the base packages needed for development environments. We install wheel first(so we can utilize it in later installs). We then uninstall py-make @@ -153,7 +158,8 @@ def install_base_environment(pip_url): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Bootstrap the development environment") - parser.add_argument("--index-url", default='https://packages.idmod.org/api/pypi/pypi-production/simple', help="Pip url to install dependencies from") + parser.add_argument("--index-url", default='https://packages.idmod.org/api/pypi/pypi-production/simple', + help="Pip url to install dependencies from") parser.add_argument("--verbose", default=False, action='store_true') args = parser.parse_args() diff --git a/dev_scripts/run_all.py b/dev_scripts/run_all.py index d4c5a435e..ce41a7b6e 100755 --- a/dev_scripts/run_all.py +++ b/dev_scripts/run_all.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -"""Runs a commnad in all the idmtools modules in this repo. +""" +Runs a command in all the idmtools modules in this repo. This is used in build processes to parallelize some operations. """ @@ -15,12 +16,13 @@ base_directory = abspath(join(dirname(__file__), '..')) modules = ['idmtools_core', 'idmtools_cli', 'idmtools_platform_comps', 'idmtools_platform_local', 'idmtools_models', 'idmtools_test', 'idmtools_platform_slurm', 'idmtools_slurm_utils', - 'idmtools_platform_file'] + 'idmtools_platform_general'] def run_command_on_all(idm_modules: List[str], command: str, parallel: bool = False, subdir: Optional[str] = None, env_override: Dict[str, str] = None): - """Runs a command in all the idmtools packages. + """ + Runs a command in all the idmtools packages. Args: idm_modules: List of modules to execute against diff --git a/dev_scripts/test_root.mk b/dev_scripts/test_root.mk index f9de72374..304121dea 100644 --- a/dev_scripts/test_root.mk +++ b/dev_scripts/test_root.mk @@ -123,24 +123,24 @@ coverage-report-view: coverage-report ## Launch coverage report. Require running $(PDS)/launch_dir_in_browser.py $(REPORT_DIR)/coverage/index.html coverage: clean ## Generate a code-coverage report - $(TEST_COMMAND) $(COVERAGE_OPTS) -m "not comps and not docker" + $(TEST_COMMAND) $(COVERAGE_OPTS) -m "not comps and not docker and not performance" coverage-smoke: clean ## Generate a code-coverage report ifneq (1, $(PARALLEL_TESTING)) # Only run these tests if Parallel Only Testing is disabled -echo "Running Serial Tests" - $(TEST_COMMAND) $(COVERAGE_OPTS) -m "smoke and serial" + $(TEST_COMMAND) $(COVERAGE_OPTS) -m "smoke and serial and not performance" endif ifneq (1, $(SERIAL_TESTING)) # Only run these tests if Serial Only Testing is disabled -echo "Running Parallel Tests" - $(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "smoke and not serial" + $(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "smoke and not serial and not performance" endif coverage-all: ## Generate a code-coverage report using all tests ifneq (1, $(PARALLEL_TESTING)) # Only run these tests if Parallel Only Testing is disabled -echo "Running Serial Tests" - $(TEST_COMMAND) $(COVERAGE_OPTS) -m "serial" + $(TEST_COMMAND) $(COVERAGE_OPTS) -m "serial and not performance" endif ifneq (1, $(SERIAL_TESTING)) # Only run these tests if Serial Only Testing is disabled -echo "Running Parallel Tests" - $(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "not serial" + $(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "not serial and not performance" endif \ No newline at end of file diff --git a/examples/platform_general/python_sims_for_fileplatform.py b/examples/platform_general/python_sims_for_fileplatform.py new file mode 100644 index 000000000..807926e7a --- /dev/null +++ b/examples/platform_general/python_sims_for_fileplatform.py @@ -0,0 +1,88 @@ +# Simple python simulation running native slurm cluster. This example is to run python script(model3.py) in slurm cluster +# which doing simple add() caculation function with 2 sweep parameters as add(a,b)=a+b). The function result will be +# printed to stdout.txt file and output/result.txt file in each simulation folder. +# Path for simulation in cluster machine: /home/username/example/suite_id/experiment_id/simulation_id +import os +import sys +from functools import partial +from typing import Any, Dict + +from idmtools.builders import SimulationBuilder +from idmtools.core.platform_factory import Platform +from idmtools.entities import Suite +from idmtools.entities.experiment import Experiment +from idmtools.entities.simulation import Simulation +from idmtools.entities.templated_simulation import TemplatedSimulations +from idmtools_models.python.json_python_task import JSONConfiguredPythonTask + +from idmtools_test import COMMON_INPUT_PATH + +# job dir should be /home/username/example +job_directory = os.path.join(os.path.expanduser('~'), "example") +# Define Slurm Platform. Note, this code can only run in slurm cluster. +platform = Platform('FILE', job_directory=job_directory) + +#Define our base task. Normally, you want to do set any assets/configurations you want across the +# all the different Simulations we are going to build for our experiment. Here we set c to 0 since we do not want to +# sweep it +task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"), + envelope="parameters", parameters=(dict(c=0))) +task.python_path = "python3" + +# now let's use this task to create a TemplatedSimulation builder. This will build new simulations from sweep builders +# we will define later. We can also use it to manipulate the base_task or the base_simulation +ts = TemplatedSimulations(base_task=task) +# We can define common metadata like tags across all the simulations using the base_simulation object +ts.base_simulation.tags['tag1'] = 1 + +# Since we have our templated simulation object now, let's define our sweeps +# To do that we need to use a builder +builder = SimulationBuilder() + +# Define an utility function that will update a single parameter at a +# # time on the model and add that param/value pair as a tag on our simulation. +def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]: + """ + This function is called during sweeping allowing us to pass the generated sweep values to our Task Configuration. + + We always receive a Simulation object. We know that simulations all have tasks and that for our particular set + of simulations they will all include JSONConfiguredPythonTask. We configure the model with calls to set_parameter + to update the config. In addition, we can return a dictionary of tags to add to the simulations so we return + the output of the 'set_parameter' call since it returns the param/value pair we set + + Args: + simulation: Simulation we are configuring + param: Param string passed to use + value: Value to set param to + + Returns: + + """ + return simulation.task.set_parameter(param, value) + +# Let's sweep the parameter 'a' for the values 0-2 +builder.add_sweep_definition(partial(param_update, param="a"), range(3)) + +# Let's sweep the parameter 'b' for the values 0-4 +builder.add_sweep_definition(partial(param_update, param="b"), range(5)) +ts.add_builder(builder) + +# Now we can create our Experiment using our template builder +experiment = Experiment.from_template(ts, name="python example") +# Add our own custom tag to experiment +experiment.tags["tag1"] = 1 +# And all files from dir at COMMON_INPUT_PATH/python/Assets folder to experiment folder +experiment.assets.add_directory(assets_directory=os.path.join(COMMON_INPUT_PATH, "python", "Assets")) + +# Create suite +suite = Suite(name='Idm Suite') +suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) +# Add experiment to the suite +suite.add_experiment(experiment) + +suite.run(platform=platform, wait_until_done=False, wait_on_done=False) +# run following command to check status +print("idmtools file " + job_directory + " status --exp-id " + experiment.id) + + + diff --git a/examples/platform_general/python_sims_for_processplatform.py b/examples/platform_general/python_sims_for_processplatform.py new file mode 100644 index 000000000..af07a79f6 --- /dev/null +++ b/examples/platform_general/python_sims_for_processplatform.py @@ -0,0 +1,86 @@ +# Simple python simulation running native slurm cluster. This example is to run python script(model3.py) in slurm cluster +# which doing simple add() caculation function with 2 sweep parameters as add(a,b)=a+b). The function result will be +# printed to stdout.txt file and output/result.txt file in each simulation folder. +# Path for simulation in cluster machine: /home/username/example/suite_id/experiment_id/simulation_id +import os +import sys +from functools import partial +from typing import Any, Dict + +from idmtools.builders import SimulationBuilder +from idmtools.core.platform_factory import Platform +from idmtools.entities import Suite +from idmtools.entities.experiment import Experiment +from idmtools.entities.simulation import Simulation +from idmtools.entities.templated_simulation import TemplatedSimulations +from idmtools_models.python.json_python_task import JSONConfiguredPythonTask + +from idmtools_test import COMMON_INPUT_PATH + +# job dir should be /home/username/example +job_directory = os.path.join(os.path.expanduser('~'), "example") +# Define Slurm Platform. Note, this code can only run in slurm cluster. +platform = Platform('PROCESS', job_directory=job_directory) + +#Define our base task. Normally, you want to do set any assets/configurations you want across the +# all the different Simulations we are going to build for our experiment. Here we set c to 0 since we do not want to +# sweep it +task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"), + envelope="parameters", parameters=(dict(c=0))) +task.python_path = "python3" + +# now let's use this task to create a TemplatedSimulation builder. This will build new simulations from sweep builders +# we will define later. We can also use it to manipulate the base_task or the base_simulation +ts = TemplatedSimulations(base_task=task) +# We can define common metadata like tags across all the simulations using the base_simulation object +ts.base_simulation.tags['tag1'] = 1 + +# Since we have our templated simulation object now, let's define our sweeps +# To do that we need to use a builder +builder = SimulationBuilder() + +# Define an utility function that will update a single parameter at a +# # time on the model and add that param/value pair as a tag on our simulation. +def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]: + """ + This function is called during sweeping allowing us to pass the generated sweep values to our Task Configuration. + + We always receive a Simulation object. We know that simulations all have tasks and that for our particular set + of simulations they will all include JSONConfiguredPythonTask. We configure the model with calls to set_parameter + to update the config. In addition, we can return a dictionary of tags to add to the simulations so we return + the output of the 'set_parameter' call since it returns the param/value pair we set + + Args: + simulation: Simulation we are configuring + param: Param string passed to use + value: Value to set param to + + Returns: + + """ + return simulation.task.set_parameter(param, value) + +# Let's sweep the parameter 'a' for the values 0-2 +builder.add_sweep_definition(partial(param_update, param="a"), range(3)) + +# Let's sweep the parameter 'b' for the values 0-4 +builder.add_sweep_definition(partial(param_update, param="b"), range(5)) +ts.add_builder(builder) + +# Now we can create our Experiment using our template builder +experiment = Experiment.from_template(ts, name="python example") +# Add our own custom tag to experiment +experiment.tags["tag1"] = 1 +# And all files from dir at COMMON_INPUT_PATH/python/Assets folder to experiment folder +experiment.assets.add_directory(assets_directory=os.path.join(COMMON_INPUT_PATH, "python", "Assets")) + +# Create suite +suite = Suite(name='Idm Suite') +suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) +# Add experiment to the suite +suite.add_experiment(experiment) + +suite.run(platform=platform, wait_until_done=True) +sys.exit(0 if experiment.succeeded else -1) + + diff --git a/idmtools_core/requirements.txt b/idmtools_core/requirements.txt index 939babfcc..02e713567 100644 --- a/idmtools_core/requirements.txt +++ b/idmtools_core/requirements.txt @@ -11,5 +11,5 @@ PyYaml>=5.3.0,<6.1 tabulate>=0.8.9,<0.10 tqdm>=4.52.0,<5 jinja2~=3.1.2 -packaging>=20.4 -pygit2>=1.4.0,<=1.12.0 \ No newline at end of file +packaging>=20.4,<22.0 +pygit2>=1.4.0,<=1.10.1 \ No newline at end of file diff --git a/idmtools_platform_file/tests/test_cli.py b/idmtools_platform_file/tests/test_cli.py deleted file mode 100644 index 17f3d51ae..000000000 --- a/idmtools_platform_file/tests/test_cli.py +++ /dev/null @@ -1,195 +0,0 @@ -import os - -import shutil -import unittest -from functools import partial -from typing import Any, Dict - -import pytest - -from idmtools.builders import SimulationBuilder -from idmtools.core.platform_factory import Platform -from idmtools.entities import Suite -from idmtools.entities.experiment import Experiment -from idmtools.entities.simulation import Simulation -from idmtools.entities.templated_simulation import TemplatedSimulations -from idmtools_models.python.json_python_task import JSONConfiguredPythonTask - -from idmtools_test import COMMON_INPUT_PATH -from idmtools_test.utils.cli import run_command, get_subcommands_from_help_result -from idmtools_test.utils.decorators import linux_only - - -@pytest.mark.serial -@linux_only -class TestCli(unittest.TestCase): - - def create_experiment(self, platform=None, a=1, b=1, retries=None, wait_until_done=False, wait_on_done=False): - task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"), - envelope="parameters", parameters=(dict(c=0))) - task.python_path = "python3" - - ts = TemplatedSimulations(base_task=task) - builder = SimulationBuilder() - - def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]: - return simulation.task.set_parameter(param, value) - - builder.add_sweep_definition(partial(param_update, param="a"), range(a)) - builder.add_sweep_definition(partial(param_update, param="b"), range(b)) - ts.add_builder(builder) - - # Now we can create our Experiment using our template builder - experiment = Experiment.from_template(ts, name=self.case_name) - # Add our own custom tag to simulation - experiment.tags["tag1"] = 1 - # And add common assets from local dir - experiment.assets.add_directory(assets_directory=os.path.join(COMMON_INPUT_PATH, "python", "Assets")) - - # Create suite - suite = Suite(name='Idm Suite') - suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) - # Add experiment to the suite - suite.add_experiment(experiment) - # Commission - suite.run(platform=platform, wait_until_done=wait_until_done, wait_on_done=wait_on_done, retries=retries) - print("suite_id: " + suite.id) - print("experiment_id: " + experiment.id) - return experiment - - @classmethod - def setUpClass(cls) -> None: - cls.job_directory = "DEST-CLI" - cls.platform = Platform('FILE', job_directory=cls.job_directory) - - @classmethod - def tearDownClass(cls) -> None: - try: - shutil.rmtree(cls.job_directory) - except: - print('Error while deleting directory') - - def setUp(self) -> None: - self.case_name = os.path.basename(__file__) + "--" + self._testMethodName - - self.experiment = self.create_experiment(self.platform, a=3, b=3) - self.suite = self.experiment.parent - self.suite_dir = self.platform.get_directory(self.suite) - - def test_subcommands_exists(self): - result = run_command('file', '--help') - print(result.stdout) - lines = get_subcommands_from_help_result(result) - self.assertListEqual(lines, ['clear-files', 'get-latest', 'get-path', 'get-status', 'status', 'status-report']) - - def test_clear_files(self): - result = run_command('file', self.job_directory, 'clear-files', '--exp-id', self.experiment.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - def test_clear_files_remove_for_sim(self): - # create file under sim first: - filename = "random.txt" - sim_dir = self.platform.get_directory(self.experiment.simulations[0]) - full_path = os.path.join(sim_dir, filename) - # Create the file using the open() function with 'w' mode - with open(full_path, 'w') as f: - pass # do nothing - self.assertTrue(os.path.isfile(full_path)) - result = run_command('file', self.job_directory, 'clear-files', '--exp-id', self.experiment.id, '--remove', - filename) - self.assertTrue(result.exit_code == 0, msg=result.output) - # check if file get removed - self.assertFalse(os.path.exists(full_path)) - print(result.stdout) - - # test remove folder under sim: - my_dir = "output" - full_my_dir = os.path.join(sim_dir, my_dir) - os.mkdir(full_my_dir) - self.assertTrue(os.path.isdir(full_my_dir)) - result = run_command('file', self.job_directory, 'clear-files', '--exp-id', self.experiment.id, '--remove', - my_dir) - self.assertTrue(result.exit_code == 0, msg=result.output) - # verify folder got deleted - self.assertFalse(os.path.isdir(full_my_dir)) - - def test_clear_files_remove_for_exp(self): - # create file under sim first: - filename = "random.txt" - exp_dir = self.platform.get_directory(self.experiment) - full_path = os.path.join(exp_dir, filename) - # Create the file using the open() function with 'w' mode - with open(full_path, 'w') as f: - pass # do nothing - self.assertTrue(os.path.isfile(full_path)) - result = run_command('file', self.job_directory, 'clear-files', '--exp-id', self.experiment.id, '--remove', - "somefile.txt") - self.assertTrue(result.exit_code == 0, msg=result.output) - # check if file get removed, it should not for experiment - self.assertTrue(os.path.exists(full_path)) - print(result.stdout) - - def test_get_latest(self): - result = run_command('file', self.job_directory, 'get-latest') - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - def test_get_path(self): - result = run_command('file', self.job_directory, 'get-path', '--exp-id', self.experiment.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - # for suite-id - result = run_command('file', self.job_directory, 'get-path', '--suite-id', self.suite.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - # for sim-id - result = run_command('file', self.job_directory, 'get-path', '--sim-id', self.experiment.simulations[0].id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - def test_get_status(self): - result = run_command('file', self.job_directory, 'get-status', '--exp-id', self.experiment.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - # for sim-id - result = run_command('file', self.job_directory, 'get-status', '--sim-id', self.experiment.simulations[0].id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - def test_status(self): - result = run_command('file', self.job_directory, 'status', '--exp-id', self.experiment.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - # for --display - result = run_command('file', self.job_directory, 'status', '--exp-id', self.experiment.id, '--display') - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - def test_status_report(self): - result = run_command('file', self.job_directory, 'status-report', '--exp-id', self.experiment.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - # suite id - result = run_command('file', self.job_directory, 'status-report', '--suite-id', self.suite.id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - # with status-filter option - result = run_command('file', self.job_directory, 'status-report', '--exp-id', self.experiment.id, - '--status-filter', '0') - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - # with display-count option - result = run_command('file', self.job_directory, 'status-report', '--exp-id', self.experiment.id, - '--display-count', 3) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - - # sim-filter option - result = run_command('file', self.job_directory, 'status-report', '--exp-id', self.experiment.id, - '--sim-filter', self.experiment.simulations[0].id) - self.assertTrue(result.exit_code == 0, msg=result.output) - print(result.stdout) - diff --git a/idmtools_platform_file/.bumpversion.cfg b/idmtools_platform_general/.bumpversion.cfg similarity index 100% rename from idmtools_platform_file/.bumpversion.cfg rename to idmtools_platform_general/.bumpversion.cfg diff --git a/idmtools_platform_file/MANIFEST.in b/idmtools_platform_general/MANIFEST.in similarity index 100% rename from idmtools_platform_file/MANIFEST.in rename to idmtools_platform_general/MANIFEST.in diff --git a/idmtools_platform_file/Makefile b/idmtools_platform_general/Makefile similarity index 100% rename from idmtools_platform_file/Makefile rename to idmtools_platform_general/Makefile diff --git a/idmtools_platform_file/README.md b/idmtools_platform_general/README.md similarity index 71% rename from idmtools_platform_file/README.md rename to idmtools_platform_general/README.md index c8366f5ce..1134b134a 100644 --- a/idmtools_platform_file/README.md +++ b/idmtools_platform_general/README.md @@ -1,6 +1,6 @@ ![Staging: idmtools-platform-file](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-file/badge.svg?branch=dev) -# idmtools-platform-file +# idmtools-platform-general @@ -16,7 +16,7 @@ ## Installing ```bash -pip install idmtools-platform-file --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple +pip install idmtools-platform-general --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple ``` # Development Tips @@ -50,12 +50,4 @@ On Windows, you can use `pymake` instead of `make` # Feature Roadmap * First Version - * Support for basic provisioning on a linux filesystem -* Second Version - * Support for Windows - * Look into documenting/testing Symbolic links on - Windows - https://www.howtogeek.com/16226/complete-guide-to-symbolic-links-symlinks-on-windows-or-linux/ - * Support for Analysis Scripts - * Support for Workitems -* Future - * Support for "True Asset Collections" \ No newline at end of file + * Support for basic provisioning on a linux filesystem \ No newline at end of file diff --git a/idmtools_platform_file/VERSION b/idmtools_platform_general/VERSION similarity index 100% rename from idmtools_platform_file/VERSION rename to idmtools_platform_general/VERSION diff --git a/idmtools_platform_file/idmtools_platform_file/__init__.py b/idmtools_platform_general/idmtools_platform_file/__init__.py similarity index 81% rename from idmtools_platform_file/idmtools_platform_file/__init__.py rename to idmtools_platform_general/idmtools_platform_file/__init__.py index c670024fd..56a23d5b5 100644 --- a/idmtools_platform_file/idmtools_platform_file/__init__.py +++ b/idmtools_platform_general/idmtools_platform_file/__init__.py @@ -1,4 +1,5 @@ -"""idmtools file platform. +""" +idmtools file platform. Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. """ diff --git a/idmtools_platform_file/idmtools_platform_file/assets/__init__.py b/idmtools_platform_general/idmtools_platform_file/assets/__init__.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/assets/__init__.py rename to idmtools_platform_general/idmtools_platform_file/assets/__init__.py diff --git a/idmtools_platform_file/idmtools_platform_file/assets/_run.sh.jinja2 b/idmtools_platform_general/idmtools_platform_file/assets/_run.sh.jinja2 similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/assets/_run.sh.jinja2 rename to idmtools_platform_general/idmtools_platform_file/assets/_run.sh.jinja2 diff --git a/idmtools_platform_file/idmtools_platform_file/assets/batch.sh.jinja2 b/idmtools_platform_general/idmtools_platform_file/assets/batch.sh.jinja2 similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/assets/batch.sh.jinja2 rename to idmtools_platform_general/idmtools_platform_file/assets/batch.sh.jinja2 diff --git a/idmtools_platform_file/idmtools_platform_file/assets/run_simulation.sh b/idmtools_platform_general/idmtools_platform_file/assets/run_simulation.sh similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/assets/run_simulation.sh rename to idmtools_platform_general/idmtools_platform_file/assets/run_simulation.sh diff --git a/idmtools_platform_file/idmtools_platform_file/cli/__init__.py b/idmtools_platform_general/idmtools_platform_file/cli/__init__.py similarity index 68% rename from idmtools_platform_file/idmtools_platform_file/cli/__init__.py rename to idmtools_platform_general/idmtools_platform_file/cli/__init__.py index 90adc9311..dd00f40ef 100644 --- a/idmtools_platform_file/idmtools_platform_file/cli/__init__.py +++ b/idmtools_platform_general/idmtools_platform_file/cli/__init__.py @@ -1,4 +1,5 @@ -"""idmtools comps cli module. +""" +idmtools file cli module. Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. """ diff --git a/idmtools_platform_file/idmtools_platform_file/cli/file.py b/idmtools_platform_general/idmtools_platform_file/cli/file.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/cli/file.py rename to idmtools_platform_general/idmtools_platform_file/cli/file.py diff --git a/idmtools_platform_file/idmtools_platform_file/file_platform.py b/idmtools_platform_general/idmtools_platform_file/file_platform.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/file_platform.py rename to idmtools_platform_general/idmtools_platform_file/file_platform.py diff --git a/idmtools_platform_file/tests/__init__.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/__init__.py similarity index 55% rename from idmtools_platform_file/tests/__init__.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/__init__.py index 84b6839e2..d65f90117 100644 --- a/idmtools_platform_file/tests/__init__.py +++ b/idmtools_platform_general/idmtools_platform_file/platform_operations/__init__.py @@ -1,4 +1,5 @@ -"""idmtools FilePlatform tests module. +""" +idmtools FilePlatform platform operations module. Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. """ diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/asset_collection_operations.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/asset_collection_operations.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/asset_collection_operations.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/asset_collection_operations.py diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/experiment_operations.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/experiment_operations.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/experiment_operations.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/experiment_operations.py diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/json_metadata_operations.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/json_metadata_operations.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/json_metadata_operations.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/json_metadata_operations.py diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/simulation_operations.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/simulation_operations.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/simulation_operations.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/simulation_operations.py diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/suite_operations.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/suite_operations.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/suite_operations.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/suite_operations.py diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/utils.py b/idmtools_platform_general/idmtools_platform_file/platform_operations/utils.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/utils.py rename to idmtools_platform_general/idmtools_platform_file/platform_operations/utils.py diff --git a/idmtools_platform_file/idmtools_platform_file/plugin_info.py b/idmtools_platform_general/idmtools_platform_file/plugin_info.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/plugin_info.py rename to idmtools_platform_general/idmtools_platform_file/plugin_info.py diff --git a/idmtools_platform_file/idmtools_platform_file/tools/__init__.py b/idmtools_platform_general/idmtools_platform_file/tools/__init__.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/tools/__init__.py rename to idmtools_platform_general/idmtools_platform_file/tools/__init__.py diff --git a/idmtools_platform_file/idmtools_platform_file/tools/status_report/__init__.py b/idmtools_platform_general/idmtools_platform_file/tools/status_report/__init__.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/tools/status_report/__init__.py rename to idmtools_platform_general/idmtools_platform_file/tools/status_report/__init__.py diff --git a/idmtools_platform_file/idmtools_platform_file/tools/status_report/status_report.py b/idmtools_platform_general/idmtools_platform_file/tools/status_report/status_report.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/tools/status_report/status_report.py rename to idmtools_platform_general/idmtools_platform_file/tools/status_report/status_report.py diff --git a/idmtools_platform_file/idmtools_platform_file/tools/status_report/utils.py b/idmtools_platform_general/idmtools_platform_file/tools/status_report/utils.py similarity index 100% rename from idmtools_platform_file/idmtools_platform_file/tools/status_report/utils.py rename to idmtools_platform_general/idmtools_platform_file/tools/status_report/utils.py diff --git a/idmtools_platform_general/idmtools_platform_process/__init__.py b/idmtools_platform_general/idmtools_platform_process/__init__.py new file mode 100644 index 000000000..c04995114 --- /dev/null +++ b/idmtools_platform_general/idmtools_platform_process/__init__.py @@ -0,0 +1,7 @@ +""" +idmtools process platform. + +Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. +""" +# flake8: noqa F821 +__version__ = "1.7.5+nightly" diff --git a/idmtools_platform_file/idmtools_platform_file/platform_operations/__init__.py b/idmtools_platform_general/idmtools_platform_process/platform_operations/__init__.py similarity index 54% rename from idmtools_platform_file/idmtools_platform_file/platform_operations/__init__.py rename to idmtools_platform_general/idmtools_platform_process/platform_operations/__init__.py index 90cd8c9c0..d10bf0403 100644 --- a/idmtools_platform_file/idmtools_platform_file/platform_operations/__init__.py +++ b/idmtools_platform_general/idmtools_platform_process/platform_operations/__init__.py @@ -1,4 +1,5 @@ -"""idmtools FilePlatform platform operations module. +""" +idmtools ProcessPlatform platform operations module. Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. """ diff --git a/idmtools_platform_general/idmtools_platform_process/platform_operations/experiment_operations.py b/idmtools_platform_general/idmtools_platform_process/platform_operations/experiment_operations.py new file mode 100644 index 000000000..4126d2240 --- /dev/null +++ b/idmtools_platform_general/idmtools_platform_process/platform_operations/experiment_operations.py @@ -0,0 +1,56 @@ +""" +Here we implement the ProcessPlatform experiment operations. + +Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. +""" +import os +from pathlib import Path +from dataclasses import dataclass +from idmtools.entities.experiment import Experiment +from idmtools_platform_file.platform_operations.experiment_operations import FilePlatformExperimentOperations +from logging import getLogger + +user_logger = getLogger('user') + + +@dataclass +class ProcessPlatformExperimentOperations(FilePlatformExperimentOperations): + """ + Experiment Operations for Process Platform. + """ + + def platform_run_item(self, experiment: Experiment, **kwargs): + """ + Run experiment. + Args: + experiment: idmtools Experiment + kwargs: keyword arguments used to expand functionality + Returns: + None + """ + # Ensure parent + experiment.parent.add_experiment(experiment) + self.platform._metas.dump(experiment.parent) + # Generate/update metadata + self.platform._metas.dump(experiment) + # Commission + self.platform.submit_job(experiment, **kwargs) + + suite_id = experiment.parent_id or experiment.suite_id + user_logger.info(f'job_directory: {Path(self.platform.job_directory).resolve()}') + user_logger.info(f'suite: {str(suite_id)}') + user_logger.info(f'experiment: {experiment.id}') + + def post_run_item(self, experiment: Experiment, **kwargs): + """ + Trigger right after commissioning experiment on platform. + + Args: + experiment: Experiment just commissioned + kwargs: keyword arguments used to expand functionality + Returns: + None + """ + super().post_run_item(experiment, **kwargs) + user_logger.info( + f'\nYou may try the following command to check simulations running status: \n idmtools file {os.path.abspath(self.platform.job_directory)} status --exp-id {experiment.id}') diff --git a/idmtools_platform_general/idmtools_platform_process/plugin_info.py b/idmtools_platform_general/idmtools_platform_process/plugin_info.py new file mode 100644 index 000000000..611286b1c --- /dev/null +++ b/idmtools_platform_general/idmtools_platform_process/plugin_info.py @@ -0,0 +1,80 @@ +""" +idmtools process platform plugin definition. + +Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. +""" +from pathlib import Path +from typing import Type, Dict +from idmtools.entities.iplatform import IPlatform +from idmtools.registry.platform_specification import example_configuration_impl, get_platform_impl, \ + get_platform_type_impl, PlatformSpecification +from idmtools.registry.plugin_specification import get_description_impl + +PROCESS_EXAMPLE_CONFIG = """ +[Process] +job_directory = /data +""" + + +class ProcessPlatformSpecification(PlatformSpecification): + """ + Process Platform Specification definition. + """ + + @get_description_impl + def get_description(self) -> str: + """ + Retrieve description. + """ + return "Provides access to the Process Platform to IDM Tools" + + @get_platform_impl + def get(self, **configuration) -> IPlatform: + """ + Build our process platform from the passed in configuration object. + + We do our import of platform here to avoid any weirdness + Args: + configuration: + Returns: + IPlatform + """ + from process_platform import ProcessPlatform + return ProcessPlatform(**configuration) + + @example_configuration_impl + def example_configuration(self): + """ + Retrieve example configuration. + """ + return PROCESS_EXAMPLE_CONFIG + + @get_platform_type_impl + def get_type(self) -> Type['ProcessPlatform']: # noqa: F821 + """ + Get type. + Returns: + Type + """ + from idmtools_platform_process.process_platform import ProcessPlatform + return ProcessPlatform + + def get_version(self) -> str: + """ + Returns the version of the plugin. + Returns: + Plugin Version + """ + from idmtools_platform_process import __version__ + return __version__ + + def get_configuration_aliases(self) -> Dict[str, Dict]: + """ + Provides configuration aliases that exist in PROCESS. + """ + config_aliases = dict( + PROCESS=dict( + job_directory=str(Path.home()) + ) + ) + return config_aliases diff --git a/idmtools_platform_general/idmtools_platform_process/process_platform.py b/idmtools_platform_general/idmtools_platform_process/process_platform.py new file mode 100644 index 000000000..fbd7c3e06 --- /dev/null +++ b/idmtools_platform_general/idmtools_platform_process/process_platform.py @@ -0,0 +1,52 @@ +""" +Here we implement the ProcessPlatform object. + +Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. +""" +import platform +import subprocess +from typing import Union, Any +from dataclasses import dataclass +from idmtools.entities.experiment import Experiment +from idmtools.entities.simulation import Simulation +from idmtools_platform_file.file_platform import FilePlatform +from idmtools_platform_process.platform_operations.experiment_operations import ProcessPlatformExperimentOperations +from logging import getLogger + +user_logger = getLogger('user') + + +@dataclass(repr=False) +class ProcessPlatform(FilePlatform): + """ + Process Platform definition. + """ + + def __post_init__(self): + super().__post_init__() + self._experiments = ProcessPlatformExperimentOperations(platform=self) + + def submit_job(self, item: Union[Experiment, Simulation], **kwargs) -> Any: + """ + Submit a Process job. + Args: + item: idmtools Experiment or Simulation + kwargs: keyword arguments used to expand functionality + Returns: + Any + """ + if platform.system() in ["Windows"]: + user_logger.warning( + "\n/!\\ WARNING: The current ProcessPlatform only support running Experiment/Simulation on Linux!") + exit(-1) + + if isinstance(item, Experiment): + working_directory = self.get_directory(item) + result = subprocess.run(['bash', 'batch.sh'], stdout=subprocess.PIPE, cwd=str(working_directory)) + r = result.stdout.decode('utf-8').strip() + return r + elif isinstance(item, Simulation): + raise NotImplementedError("submit_job directly for simulation is not implemented on ProcessPlatform.") + else: + raise NotImplementedError( + f"Submit job is not implemented for {item.__class__.__name__} on ProcessPlatform.") diff --git a/idmtools_platform_file/requirements.txt b/idmtools_platform_general/requirements.txt similarity index 100% rename from idmtools_platform_file/requirements.txt rename to idmtools_platform_general/requirements.txt diff --git a/idmtools_platform_file/setup.py b/idmtools_platform_general/setup.py similarity index 74% rename from idmtools_platform_file/setup.py rename to idmtools_platform_general/setup.py index 2386779a7..26b9b6837 100644 --- a/idmtools_platform_file/setup.py +++ b/idmtools_platform_general/setup.py @@ -1,7 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""The setup script for the idmtools_platform_file platform, for users who use the COMPS platform for idmtools.""" +""" +The setup script for the idmtools_platform_file platform, for users who use the COMPS platform for idmtools. +""" from setuptools import setup, find_packages with open('README.md', encoding='utf-8') as readme_file: @@ -11,7 +13,8 @@ requirements = requirements_file.read().split("\n") build_requirements = ['flake8', 'coverage', 'bump2version', 'twine', "natsort"] -test_requirements = ['pytest', 'pytest-runner', 'matplotlib', 'pytest-timeout', 'pytest-cache', 'pytest-lazy-fixture'] + build_requirements +test_requirements = ['pytest', 'pytest-runner', 'matplotlib', 'pytest-timeout', 'pytest-cache', + 'pytest-lazy-fixture'] + build_requirements extras = dict(test=test_requirements, dev=['Pympler'], packaging=build_requirements) @@ -34,17 +37,18 @@ classifiers=[ 'Framework:: IDM-Tools :: Platform', ], - description="File platform for IDM-Tools", + description="General platform for IDM-Tools", install_requires=requirements, long_description=readme, include_package_data=True, keywords='modeling, IDM', - name='idmtools_platform_file', + name='idmtools_platform_general', packages=find_packages(), test_suite='tests', entry_points={"idmtools_platform": [ - "idmtools_platform_file = idmtools_platform_file.plugin_info:FilePlatformSpecification"], - "idmtools_cli.cli_plugins": ["file=idmtools_platform_file.cli.file:file"] + "idmtools_platform_file = idmtools_platform_file.plugin_info:FilePlatformSpecification", + "idmtools_platform_process = idmtools_platform_process.plugin_info:ProcessPlatformSpecification"], + "idmtools_cli.cli_plugins": ["file = idmtools_platform_file.cli.file:file"] }, extras_require=extras, url='https://github.com/InstituteforDiseaseModeling/idmtools', diff --git a/idmtools_platform_file/tests/Makefile b/idmtools_platform_general/tests/Makefile similarity index 100% rename from idmtools_platform_file/tests/Makefile rename to idmtools_platform_general/tests/Makefile diff --git a/idmtools_platform_general/tests/__init__.py b/idmtools_platform_general/tests/__init__.py new file mode 100644 index 000000000..1d47907b2 --- /dev/null +++ b/idmtools_platform_general/tests/__init__.py @@ -0,0 +1,5 @@ +""" +idmtools FilePlatform & ProcessPlatform tests module. + +Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved. +""" diff --git a/idmtools_platform_general/tests/input/hello.sh b/idmtools_platform_general/tests/input/hello.sh new file mode 100644 index 000000000..36d595fad --- /dev/null +++ b/idmtools_platform_general/tests/input/hello.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello World" \ No newline at end of file diff --git a/idmtools_platform_file/tests/pytest.ini b/idmtools_platform_general/tests/pytest.ini similarity index 100% rename from idmtools_platform_file/tests/pytest.ini rename to idmtools_platform_general/tests/pytest.ini diff --git a/idmtools_platform_file/tests/test_json_metadata_operations.py b/idmtools_platform_general/tests/test_json_metadata_operations.py similarity index 100% rename from idmtools_platform_file/tests/test_json_metadata_operations.py rename to idmtools_platform_general/tests/test_json_metadata_operations.py diff --git a/idmtools_platform_file/tests/test_generated_files_and_directories.py b/idmtools_platform_general/tests/test_platform_file.py similarity index 98% rename from idmtools_platform_file/tests/test_generated_files_and_directories.py rename to idmtools_platform_general/tests/test_platform_file.py index 5a33e55b1..4966c07bd 100644 --- a/idmtools_platform_file/tests/test_generated_files_and_directories.py +++ b/idmtools_platform_general/tests/test_platform_file.py @@ -27,7 +27,7 @@ @pytest.mark.serial @linux_only -class TestFilesAndDirectories(ITestWithPersistence): +class TestFilePlatform(ITestWithPersistence): def create_experiment(self, platform=None, a=1, b=1, retries=None, wait_until_done=False, wait_on_done=False): task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"), @@ -158,7 +158,7 @@ def test_generated_scripts(self): self.assertEqual(contents['item_type'], 'Experiment') self.assertEqual(contents['status'], 'CREATED') self.assertEqual(len(contents['assets']), 6) - self.assertEqual(contents['name'], 'test_generated_files_and_directories.py--test_generated_scripts') + self.assertEqual(contents['name'], 'test_platform_file.py--test_generated_scripts') self.assertEqual(contents['task_type'], 'idmtools.entities.command_task.CommandTask') # verify ids in metadata.json for simulation, also verify sweep parameter in config.json file diff --git a/idmtools_platform_general/tests/test_platform_process.py b/idmtools_platform_general/tests/test_platform_process.py new file mode 100644 index 000000000..f792a9665 --- /dev/null +++ b/idmtools_platform_general/tests/test_platform_process.py @@ -0,0 +1,235 @@ +import json +import os +import pathlib +from functools import partial +from typing import Any, Dict + +import numpy as np +import pandas as pd +import pytest +from pathlib import Path + +from idmtools.builders import SimulationBuilder +from idmtools.core import ItemType +from idmtools.core.platform_factory import Platform +from idmtools.entities import Suite +from idmtools.entities.experiment import Experiment +from idmtools.entities.simulation import Simulation +from idmtools.entities.templated_simulation import TemplatedSimulations +from idmtools_models.python.json_python_task import JSONConfiguredPythonTask + +from idmtools_test import COMMON_INPUT_PATH +from idmtools_test.utils.decorators import linux_only +from idmtools_test.utils.itest_with_persistence import ITestWithPersistence + + +@pytest.mark.serial +@linux_only +class TestProcessPlatform(ITestWithPersistence): + + def create_experiment(self, platform=None, a=1, b=1, retries=None, wait_on_done=True): + task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"), + envelope="parameters", parameters=(dict(c=0))) + task.python_path = "python3" + + ts = TemplatedSimulations(base_task=task) + builder = SimulationBuilder() + + def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]: + return simulation.task.set_parameter(param, value) + + builder.add_sweep_definition(partial(param_update, param="a"), range(a)) + builder.add_sweep_definition(partial(param_update, param="b"), range(b)) + ts.add_builder(builder) + + # Now we can create our Experiment using our template builder + experiment = Experiment.from_template(ts, name=self.case_name) + # Add our own custom tag to simulation + experiment.tags["tag1"] = 1 + # And add common assets from local dir + experiment.assets.add_directory(assets_directory=os.path.join(COMMON_INPUT_PATH, "python", "Assets")) + + # Create suite + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + # Commission + suite.run(platform=platform, wait_on_done=wait_on_done, retries=retries) + print("suite_id: " + suite.id) + print("experiment_id: " + experiment.id) + return experiment + + def setUp(self) -> None: + self.case_name = os.path.basename(__file__) + "--" + self._testMethodName + self.job_directory = "DEST" + self.platform = Platform('PROCESS', job_directory=self.job_directory) + + def test_suite_experiment_simulation(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + # first verify files and dirs under suite + suite = experiment.parent + suite_dir = self.platform.get_directory(suite) + self.assertEqual(Path(suite_dir), Path(f'{self.job_directory}/{suite.id}')) + files = [] + dirs = [] + for (dirpath, dirnames, filenames) in os.walk(suite_dir): + files.extend(filenames) + dirs.extend(dirnames) + break + self.assertSetEqual(set(files), set(["metadata.json"])) + self.assertEqual(dirs[0], experiment.id) + # second verify files and dirs under experiment + experiment_dir = self.platform.get_directory(experiment) + files = [] + for (dirpath, dirnames, filenames) in os.walk(experiment_dir): + files.extend(filenames) + break + self.assertSetEqual(set(files), set(["metadata.json", "run_simulation.sh", "batch.sh", "stdout.txt", "stderr.txt"])) + + # verify all files under simulations + self.assertEqual(experiment.simulation_count, 9) + count = 0 + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + asserts_dir = simulation_dir.joinpath("Assets") + files = [] + for (dirpath, dirnames, filenames) in os.walk(simulation_dir): + if "Assets" in dirnames: + # verify Assets folder under simulation is symlink and it link to experiment's Assets + self.assertTrue(os.path.islink(asserts_dir)) + target_link = os.readlink(asserts_dir) + self.assertEqual(os.path.basename(pathlib.Path(target_link).parent), experiment.id) + count = count + 1 + files.extend(filenames) + self.assertSetEqual(set(files), set(["metadata.json", "_run.sh", "config.json", "stdout.txt", "stderr.txt", "job_status.txt", "result.txt"])) + self.assertEqual(count, 9) # make sure we found total 9 symlinks for Assets folder + + def test_generated_scripts(self): + platform = Platform('PROCESS', job_directory=self.job_directory, retries=5) + experiment = self.create_experiment(platform=platform, a=5, b=5) + experiment_dir = self.platform.get_directory(experiment) + # verify sbatch.sh script content in experiment level + with open(os.path.join(experiment_dir, 'batch.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn( + 'find $(pwd) -maxdepth 2 -name "_run.sh" -print0 | xargs -0 -I% dirname % | xargs -d "\\n" -I% bash -c \'cd $(pwd) && $(pwd)/run_simulation.sh % 1>> stdout.txt 2>> stderr.txt\'', + contents) + + # verify run_simulation.sh script content in experiment level + with open(os.path.join(experiment_dir, 'run_simulation.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn( + "JOB_DIRECTORY=$1\necho \"enter directory: \'$JOB_DIRECTORY\'\"", contents) + self.assertIn("cd $JOB_DIRECTORY", contents) + self.assertIn("bash _run.sh 1> stdout.txt 2> stderr.txt", contents) + + # verify _run.sh script content under simulation level + simulation_ids = [] + for simulation in experiment.simulations: + simulation_ids.append(simulation.id) + simulation_dir = platform.get_directory(simulation) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + + self.assertIn("python3 Assets/model3.py --config config.json", contents) + + # verify ids in metadata.json for suite + suite = experiment.suite + suite_dir = platform.get_directory(suite) + with open(os.path.join(suite_dir, 'metadata.json'), 'r') as j: + contents = json.loads(j.read()) + self.assertEqual(contents['_uid'], suite.id) + self.assertEqual(contents['uid'], suite.id) + self.assertEqual(contents['id'], suite.id) + self.assertEqual(contents['parent_id'], None) + self.assertEqual(contents['tags'], {'name': 'suite_tag', 'idmtools': '123'}) + self.assertEqual(contents['item_type'], 'Suite') + + # verify ids in metadata.json for experiment + with open(os.path.join(experiment_dir, 'metadata.json'), 'r') as j: + contents = json.loads(j.read()) + self.assertEqual(contents['_uid'], experiment.id) + self.assertEqual(contents['uid'], experiment.id) + self.assertEqual(contents['id'], experiment.id) + self.assertEqual(contents['parent_id'], suite.id) + self.assertEqual(contents['item_type'], 'Experiment') + self.assertEqual(contents['status'], 'CREATED') + self.assertEqual(len(contents['assets']), 6) + self.assertEqual(contents['name'], 'test_platform_process.py--test_generated_scripts') + self.assertEqual(contents['task_type'], 'idmtools.entities.command_task.CommandTask') + + # verify ids in metadata.json for simulation, also verify sweep parameter in config.json file + for simulation in experiment.simulations: + simulation_dir = platform.get_directory(simulation) + with open(os.path.join(simulation_dir, 'metadata.json'), 'r') as j: + contents = json.loads(j.read()) + self.assertEqual(contents['_uid'], simulation.id) + self.assertEqual(contents['uid'], simulation.id) + self.assertEqual(contents['id'], simulation.id) + self.assertEqual(contents['parent_id'], experiment.id) + self.assertEqual(contents['task']['command'], 'python3 Assets/model3.py --config config.json') + with open(os.path.join(simulation_dir, 'config.json'), 'r') as j: + config_contents = json.loads(j.read()) + self.assertDictEqual(contents['task']['parameters'], config_contents['parameters']) + + def test_create_sim_directory_map(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + exp_map = self.platform.create_sim_directory_map(experiment.id, item_type=ItemType.EXPERIMENT) + sims_map_dict = {} + for sim in experiment.simulations: + sim_map = self.platform.create_sim_directory_map(sim.id, item_type=ItemType.SIMULATION) + self.assertTrue(sim_map[sim.id], + os.path.join(self.job_directory, experiment.parent_id, experiment.id, sim.id)) + sims_map_dict.update({sim.id: sim_map[sim.id]}) + self.assertDictEqual(exp_map, sims_map_dict) + self.assertTrue(len(exp_map) == 9) + + def test_create_sim_directory_df(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + exp_df = self.platform.create_sim_directory_df(experiment.id) + sims_df = pd.DataFrame() + for sim in experiment.simulations: + sim_map = self.platform.create_sim_directory_map(sim.id, item_type=ItemType.SIMULATION) + sim_tags = sim.tags + sim_df = pd.DataFrame(sim_tags, index=[0]) + sim_df['simid'] = sim.id + sim_df['outpath'] = sim_map[sim.id] + sims_df = pd.concat([sims_df, sim_df], ignore_index=True) + self.assertTrue(np.all(exp_df.sort_values('simid').values == sims_df.sort_values('simid').values)) + self.assertTrue(exp_df.shape == (9, 6)) + + def test_create_sim_directory_csv(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + self.platform.save_sim_directory_df_to_csv(experiment.id) + exp_df = self.platform.create_sim_directory_df(experiment.id) + import csv + sim_list = [] + with open(f"{experiment.id}.csv", newline='') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + sim_list.append(row) + self.assertTrue(exp_df.values.tolist(), sim_list) + self.assertTrue(len(sim_list) == 9) + # cleanup + os.remove(f"{experiment.id}.csv") + + def test_platform_delete_experiment(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + self.platform._experiments.platform_delete(experiment.id) + # make sure we don't delete suite in this case + self.assertTrue(os.path.exists(os.path.join(self.job_directory, experiment.parent_id))) + # make sure we only delete experiment folder under suite + self.assertFalse(os.path.exists(os.path.join(self.job_directory, experiment.parent_id, experiment.id))) + with self.assertRaises(RuntimeError) as context: + self.platform.get_item(experiment.id, item_type=ItemType.EXPERIMENT, raw=True) + self.assertTrue(f"Not found Experiment with id '{experiment.id}'" in str(context.exception.args[0])) + + def test_platform_delete_suite(self): + experiment = self.create_experiment(self.platform, a=3, b=3) + self.platform._suites.platform_delete(experiment.parent_id) + # make sure we delete suite folder + self.assertFalse(os.path.exists(os.path.join(self.job_directory, experiment.parent_id))) + with self.assertRaises(RuntimeError) as context: + self.platform.get_item(experiment.parent_id, item_type=ItemType.SUITE, raw=True) + self.assertTrue(f"Not found Suite with id '{experiment.parent_id}'" in str(context.exception.args[0])) diff --git a/idmtools_platform_general/tests/test_singularity_fileplatform.py b/idmtools_platform_general/tests/test_singularity_fileplatform.py new file mode 100644 index 000000000..71075194c --- /dev/null +++ b/idmtools_platform_general/tests/test_singularity_fileplatform.py @@ -0,0 +1,65 @@ +import os + +import pytest + +from idmtools.core.platform_factory import Platform +from idmtools.entities import Suite +from idmtools.entities.command_task import CommandTask +from idmtools.entities.experiment import Experiment + +from idmtools_test.utils.decorators import linux_only +from idmtools_test.utils.itest_with_persistence import ITestWithPersistence + + +@pytest.mark.serial +@linux_only +class TestSingularity(ITestWithPersistence): + + def setUp(self) -> None: + self.case_name = os.path.basename(__file__) + "--" + self._testMethodName + self.job_directory = "DEST" + self.platform = Platform('FILE', job_directory=self.job_directory) + + def test_command_build_singularity(self): + command = "singularity exec my.sif Assets/hello.sh" + task = CommandTask(command=command) + task.common_assets.add_asset("input/hello.sh") + + # create experiment from task + experiment = Experiment.from_task(task, name="run_task_in_singularity") + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + suite.run(platform=self.platform, wait_on_done=False, wait_until_done=False) + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + exe = simulation_dir.joinpath("Assets/hello.sh") + self.assertTrue(os.access(exe, os.X_OK)) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn(command, contents) + + # To test command line with extra command option for singularity case in _run.sh for each simulation + # note, we do not need to test run successfully, since we do not have actually sif file to test + def test_extra_command_build_singularity(self): + command = "Assets/hello.sh" # assume hello.sh is our executable + task = CommandTask(command=command) + task.sif_path = "my_sif.sif" + task.common_assets.add_asset("input/hello.sh") + task.command._options.update({"--python-script-path": "./Assets/python"}) + + # create experiment from task + experiment = Experiment.from_task(task, name="run_task_in_singularity") + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + suite.run(platform=self.platform, wait_on_done=False, wait_until_done=False) + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + exe = simulation_dir.joinpath(command) + self.assertTrue(os.access(exe, os.X_OK)) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn("singularity exec " + task.sif_path + " " + command + " --python-script-path ./Assets/python", contents) diff --git a/idmtools_platform_general/tests/test_singularity_processplatform.py b/idmtools_platform_general/tests/test_singularity_processplatform.py new file mode 100644 index 000000000..84466028e --- /dev/null +++ b/idmtools_platform_general/tests/test_singularity_processplatform.py @@ -0,0 +1,65 @@ +import os + +import pytest + +from idmtools.core.platform_factory import Platform +from idmtools.entities import Suite +from idmtools.entities.command_task import CommandTask +from idmtools.entities.experiment import Experiment + +from idmtools_test.utils.decorators import linux_only +from idmtools_test.utils.itest_with_persistence import ITestWithPersistence + + +@pytest.mark.serial +@linux_only +class TestSingularity(ITestWithPersistence): + + def setUp(self) -> None: + self.case_name = os.path.basename(__file__) + "--" + self._testMethodName + self.job_directory = "DEST" + self.platform = Platform('PROCESS', job_directory=self.job_directory) + + def test_command_build_singularity(self): + command = "singularity exec my.sif Assets/hello.sh" + task = CommandTask(command=command) + task.common_assets.add_asset("input/hello.sh") + + # create experiment from task + experiment = Experiment.from_task(task, name="run_task_in_singularity") + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + suite.run(platform=self.platform) + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + exe = simulation_dir.joinpath("Assets/hello.sh") + self.assertTrue(os.access(exe, os.X_OK)) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn(command, contents) + + # To test command line with extra command option for singularity case in _run.sh for each simulation + # note, we do not need to test run successfully, since we do not have actually sif file to test + def test_extra_command_build_singularity(self): + command = "Assets/hello.sh" # assume hello.sh is our executable + task = CommandTask(command=command) + task.sif_path = "my_sif.sif" + task.common_assets.add_asset("input/hello.sh") + task.command._options.update({"--python-script-path": "./Assets/python"}) + + # create experiment from task + experiment = Experiment.from_task(task, name="run_task_in_singularity") + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + suite.run(platform=self.platform) + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + exe = simulation_dir.joinpath(command) + self.assertTrue(os.access(exe, os.X_OK)) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn("singularity exec " + task.sif_path + " " + command + " --python-script-path ./Assets/python", contents) diff --git a/idmtools_platform_slurm/tests/test_singularity.py b/idmtools_platform_slurm/tests/test_singularity.py index 89303eb2d..e00ff4b9f 100644 --- a/idmtools_platform_slurm/tests/test_singularity.py +++ b/idmtools_platform_slurm/tests/test_singularity.py @@ -62,3 +62,27 @@ def testSingularity(self): with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: contents = fpr.read() self.assertIn(command, contents) + + def test_extra_command_build_singularity(self): + command = "Assets/hello.sh" # assume hello.sh is our executable + task = CommandTask(command=command) + task.sif_path = "my_sif.sif" + task.common_assets.add_asset("input/hello.sh") + task.command._options.update({"--python-script-path": "./Assets/python"}) + + # create experiment from task + experiment = Experiment.from_task(task, name="run_task_in_singularity") + suite = Suite(name='Idm Suite') + suite.update_tags({'name': 'suite_tag', 'idmtools': '123'}) + # Add experiment to the suite + suite.add_experiment(experiment) + suite.run(platform=self.platform, wait_until_done=False, wait_on_done=False, + max_running_jobs=10, + retries=5, dry_run=True) + for simulation in experiment.simulations: + simulation_dir = self.platform.get_directory(simulation) + exe = simulation_dir.joinpath(command) + self.assertTrue(os.access(exe, os.X_OK)) + with open(os.path.join(simulation_dir, '_run.sh'), 'r') as fpr: + contents = fpr.read() + self.assertIn("singularity exec " + task.sif_path + " " + command + " --python-script-path ./Assets/python", contents) \ No newline at end of file