Merge pull request #2044 from InstituteforDiseaseModeling/platform_ge…

…neral

Platform-General: implementation of ProcessPlatform

.github/workflows/run_dev_install_all_tests.yml

@@ -93,16 +93,16 @@ jobs:
         with:
           name: idmtools_slurm_utils_test_results
           path: idmtools_slurm_utils/tests/reports/
-      - name: run idmtools_platform_file tests
+      - name: run idmtools_platform_general tests
         run: |
-            cd idmtools_platform_file
+            cd idmtools_platform_general
             make test-all
-      - name: Upload idmtools_platform_file all test results
+      - name: Upload idmtools_platform_general all test results
         uses: actions/upload-artifact@v2
         if: failure()
         with:
-          name: idmtools_platform_file_test_results
-          path: idmtools_platform_file/tests/reports/
+          name: idmtools_platform_general_test_results
+          path: idmtools_platform_general/tests/reports/
       - name: run idmtools_platform_comps tests
         run: |
           cd idmtools_platform_comps

.github/workflows/run_pip_prod_all_tests.yml

@@ -89,6 +89,16 @@ jobs:
         with:
           name: idmtools_slurm_utils_test_results
           path: idmtools_slurm_utils/tests/reports/
+      - name: run idmtools_platform_general tests
+        run: |
+          cd idmtools_platform_general
+          make test-all
+      - name: Upload idmtools_platform_general all test results
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: idmtools_platform_general_test_results
+          path: idmtools_platform_general/tests/reports/
       - name: run idmtools_platform_comps tests
         run: |
           cd idmtools_platform_comps

.github/workflows/run_pip_stage_all_tests.yml

@@ -93,6 +93,16 @@ jobs:
         with:
           name: idmtools_slurm_utils_test_results
           path: idmtools_slurm_utils/tests/reports/
+      - name: run idmtools_platform_general tests
+        run: |
+          cd idmtools_platform_general
+          make test-all
+      - name: Upload idmtools_platform_general all test results
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: idmtools_platform_general_test_results
+          path: idmtools_platform_general/tests/reports/
       - name: run idmtools_platform_comps tests
         run: |
           cd idmtools_platform_comps
@@ -104,13 +114,3 @@ jobs:
           name: idmtools_platform_comps_test_results
           path: idmtools_platform_comps/tests/reports/
 
-      - name: run idmtools_platform_file tests
-        run: |
-          cd idmtools_platform_file
-          make test-all
-      - name: Upload idmtools_platform_file all test results
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: idmtools_platform_file_test_results
-          path: idmtools_platform_file/tests/reports/

dev_scripts/bootstrap.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-"""This scripts aids in setup of development environments.
+"""
+This scripts aids in setup of development environments.
 
 The script installs all the local packages
 defined by packages using development installs.
@@ -23,12 +24,10 @@
 from os.path import abspath, join, dirname
 from typing import List, Generator
 
-
-# on windows virtual env is not populated through pymake
+# on windowns virtual env is not populated through pymake
 if sys.platform == "win32" and 'VIRTUAL_ENV' in os.environ:
     sys.path.insert(0, os.environ['VIRTUAL_ENV'] + "\\Lib\\site-packages")
 
-
 script_dir = abspath(dirname(__file__))
 base_directory = abspath(join(dirname(__file__), '..'))
 
@@ -46,15 +45,16 @@
     idmtools_platform_comps=data_class_default,
     idmtools_models=data_class_default,
     idmtools_platform_slurm=data_class_default,
-    idmtools_platform_file=data_class_default,
+    idmtools_platform_general=data_class_default,
     idmtools_slurm_utils=[],
     idmtools_test=[]
 )
 logger = getLogger("bootstrap")
 
 
 def execute(cmd: List['str'], cwd: str = base_directory, ignore_error: bool = False) -> Generator[str, None, None]:
-    """Runs a command and filters output.
+    """
+    Runs a command and filters output.
 
     Args:
         cmd: Command to run
@@ -77,15 +77,16 @@ def execute(cmd: List['str'], cwd: str = base_directory, ignore_error: bool = Fa
 
 
 def process_output(output_line: str):
-    """Process output line for display.
+    """
+    Process output line for display.
 
     This function adds coloring, filters output, and strips non-ascii characters(Docker builds have some odd characters)
 
     Args:
         output_line: Output line
 
     Returns:
-        None. Instead prints to log level on screen
+        None. Instead of prints to log level on screen
     """
     # catch errors where possible
     if "FAILED [" in output_line:
@@ -100,10 +101,11 @@ def process_output(output_line: str):
 
 
 def install_dev_packages(pip_url):
-    """Install the development packages.
+    """
+    Install the development packages.
 
     This loops through all our idmtools packages and runs pip install -e . on each package
-    It also runs a pip install -r requirements from the  docs directory.
+    It also runs a pip install -r requirements from the  doc directory.
 
     Args:
         pip_url: Url to install package from
@@ -116,18 +118,21 @@ def install_dev_packages(pip_url):
         extras_str = f"[{','.join(extras)}]" if extras else ''
         logger.info(f'Installing {package} with extras: {extras_str if extras_str else "None"} from {base_directory}')
         try:
-            for line in execute(["pip3", "install", "-e", f".{extras_str}", f"--extra-index-url={pip_url}"], cwd=join(base_directory, package)):
+            for line in execute(["pip3", "install", "-e", f".{extras_str}", f"--extra-index-url={pip_url}"],
+                                cwd=join(base_directory, package)):
                 process_output(line)
         except subprocess.CalledProcessError as e:
             logger.critical(f'{package} installed failed using {e.cmd} did not succeed')
             result = e.returncode
             logger.debug(f'Return Code: {result}')
-    for line in execute(["pip3", "install", "-r", "requirements.txt", f"--extra-index-url={pip_url}"], cwd=join(base_directory, 'docs')):
+    for line in execute(["pip3", "install", "-r", "requirements.txt", f"--extra-index-url={pip_url}"],
+                        cwd=join(base_directory, 'docs')):
         process_output(line)
 
 
 def install_base_environment(pip_url):
-    """Installs the base packages needed for development environments.
+    """
+    Installs the base packages needed for development environments.
 
     We install wheel first(so we can utilize it in later installs).
     We then uninstall py-make
@@ -153,7 +158,8 @@ def install_base_environment(pip_url):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Bootstrap the development environment")
-    parser.add_argument("--index-url", default='https://packages.idmod.org/api/pypi/pypi-production/simple', help="Pip url to install dependencies from")
+    parser.add_argument("--index-url", default='https://packages.idmod.org/api/pypi/pypi-production/simple',
+                        help="Pip url to install dependencies from")
     parser.add_argument("--verbose", default=False, action='store_true')
 
     args = parser.parse_args()

dev_scripts/run_all.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-"""Runs a commnad in all the idmtools modules in this repo.
+"""
+Runs a command in all the idmtools modules in this repo.
 
 This is used in build processes to parallelize some operations.
 """
@@ -15,12 +16,13 @@
 base_directory = abspath(join(dirname(__file__), '..'))
 modules = ['idmtools_core', 'idmtools_cli', 'idmtools_platform_comps', 'idmtools_platform_local',
            'idmtools_models', 'idmtools_test', 'idmtools_platform_slurm', 'idmtools_slurm_utils',
-           'idmtools_platform_file']
+           'idmtools_platform_general']
 
 
 def run_command_on_all(idm_modules: List[str], command: str, parallel: bool = False, subdir: Optional[str] = None,
                        env_override: Dict[str, str] = None):
-    """Runs a command in all the idmtools packages.
+    """
+    Runs a command in all the idmtools packages.
 
     Args:
         idm_modules: List of modules to execute against

dev_scripts/test_root.mk

@@ -123,24 +123,24 @@ coverage-report-view: coverage-report ## Launch coverage report. Require running
 	$(PDS)/launch_dir_in_browser.py $(REPORT_DIR)/coverage/index.html
 
 coverage: clean ## Generate a code-coverage report
-	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "not comps and not docker"
+	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "not comps and not docker and not performance"
 
 coverage-smoke: clean ## Generate a code-coverage report
 ifneq (1, $(PARALLEL_TESTING)) # Only run these tests if Parallel Only Testing is disabled
 	-echo "Running Serial Tests"
-	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "smoke and serial"
+	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "smoke and serial and not performance"
 endif
 ifneq (1, $(SERIAL_TESTING)) # Only run these tests if Serial Only Testing is disabled
 	-echo "Running Parallel Tests"
-	$(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "smoke and not serial"
+	$(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "smoke and not serial and not performance"
 endif
 
 coverage-all: ## Generate a code-coverage report using all tests
 ifneq (1, $(PARALLEL_TESTING)) # Only run these tests if Parallel Only Testing is disabled
 	-echo "Running Serial Tests"
-	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "serial"
+	$(TEST_COMMAND) $(COVERAGE_OPTS) -m "serial and not performance"
 endif
 ifneq (1, $(SERIAL_TESTING)) # Only run these tests if Serial Only Testing is disabled
 	-echo "Running Parallel Tests"
-	$(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "not serial"
+	$(TEST_COMMAND) $(COVERAGE_OPTS) -n $(PARALLEL_TEST_COUNT) -m "not serial and not performance"
 endif

examples/platform_general/python_sims_for_fileplatform.py

@@ -0,0 +1,88 @@
+# Simple python simulation running native slurm cluster. This example is to run python script(model3.py) in slurm cluster
+# which doing simple add() caculation function with 2 sweep parameters as add(a,b)=a+b). The function result will be
+# printed to stdout.txt file and output/result.txt file in each simulation folder.
+# Path for simulation in cluster machine: /home/username/example/suite_id/experiment_id/simulation_id
+import os
+import sys
+from functools import partial
+from typing import Any, Dict
+
+from idmtools.builders import SimulationBuilder
+from idmtools.core.platform_factory import Platform
+from idmtools.entities import Suite
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+from idmtools.entities.templated_simulation import TemplatedSimulations
+from idmtools_models.python.json_python_task import JSONConfiguredPythonTask
+
+from idmtools_test import COMMON_INPUT_PATH
+
+# job dir should be /home/username/example
+job_directory = os.path.join(os.path.expanduser('~'), "example")
+# Define Slurm Platform. Note, this code can only run in slurm cluster.
+platform = Platform('FILE', job_directory=job_directory)
+
+#Define our base task. Normally, you want to do set any assets/configurations you want across the
+# all the different Simulations we are going to build for our experiment. Here we set c to 0 since we do not want to
+# sweep it
+task = JSONConfiguredPythonTask(script_path=os.path.join(COMMON_INPUT_PATH, "python", "model3.py"),
+                                envelope="parameters", parameters=(dict(c=0)))
+task.python_path = "python3"
+
+# now let's use this task to create a TemplatedSimulation builder. This will build new simulations from sweep builders
+# we will define later. We can also use it to manipulate the base_task or the base_simulation
+ts = TemplatedSimulations(base_task=task)
+# We can define common metadata like tags across all the simulations using the base_simulation object
+ts.base_simulation.tags['tag1'] = 1
+
+# Since we have our templated simulation object now, let's define our sweeps
+# To do that we need to use a builder
+builder = SimulationBuilder()
+
+# Define an utility function that will update a single parameter at a
+# # time on the model and add that param/value pair as a tag on our simulation.
+def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]:
+    """
+    This function is called during sweeping allowing us to pass the generated sweep values to our Task Configuration.
+
+    We always receive a Simulation object. We know that simulations all have tasks and that for our particular set
+    of simulations they will all include JSONConfiguredPythonTask. We configure the model with calls to set_parameter
+    to update the config. In addition, we can return a dictionary of tags to add to the simulations so we return
+    the output of the 'set_parameter' call since it returns the param/value pair we set
+
+    Args:
+        simulation: Simulation we are configuring
+        param: Param string passed to use
+        value: Value to set param to
+
+    Returns:
+
+    """
+    return simulation.task.set_parameter(param, value)
+
+# Let's sweep the parameter 'a' for the values 0-2
+builder.add_sweep_definition(partial(param_update, param="a"), range(3))
+
+# Let's sweep the parameter 'b' for the values 0-4
+builder.add_sweep_definition(partial(param_update, param="b"), range(5))
+ts.add_builder(builder)
+
+# Now we can create our Experiment using our template builder
+experiment = Experiment.from_template(ts, name="python example")
+# Add our own custom tag to experiment
+experiment.tags["tag1"] = 1
+# And all files from dir at COMMON_INPUT_PATH/python/Assets folder to experiment folder
+experiment.assets.add_directory(assets_directory=os.path.join(COMMON_INPUT_PATH, "python", "Assets"))
+
+# Create suite
+suite = Suite(name='Idm Suite')
+suite.update_tags({'name': 'suite_tag', 'idmtools': '123'})
+# Add experiment to the suite
+suite.add_experiment(experiment)
+
+suite.run(platform=platform, wait_until_done=False, wait_on_done=False)
+# run following command to check status
+print("idmtools file " + job_directory + " status --exp-id " + experiment.id)
+
+
+