v1.3.0: enable scripts to switch Comps environments; update with late…

…st dependencies; (#66)

* work around pulp error
* update the code to work with the Latest Emodpy_Malaria and EMOD.
* add the feature to switch to a different Comps environment(ex: Production to Staging, Calculon to Nibbler)
* bump version to 1.3.0 and update baseline and validation pdf report

create_plots/helpers_likelihood_and_metrics.py

@@ -15,7 +15,7 @@
 import warnings
 import numpy as np
 from plotnine import ggplot, aes, geom_point, xlab, ylab, coord_fixed, geom_abline, theme_classic, themes, \
-    ggtitle, geom_smooth
+    ggtitle, geom_smooth, element_text
 import collections
 
 
@@ -230,7 +230,7 @@ def corr_ref_sim_points(combined_df):
           + geom_abline(slope=1, intercept=0, color='grey', alpha=0.5)
           + geom_smooth(method="lm", fill=None, se=False, alpha=0.5, size=0.5)
           + theme_classic()
-          + themes.theme(plot_title=themes.element_text(size=12)))
+          + themes.theme(plot_title=element_text(size=12)))
 
     # create data frame with information about linear regression and correlation for each Site
     lm_summary = collections.defaultdict(list)
@@ -296,7 +296,7 @@ def corr_ref_deriv_sim_points(combined_df):
           + geom_abline(slope=1, intercept=0, color='grey', alpha=0.5)
           # + geom_smooth(method="lm", fill=None, se=False, alpha=0.5, size=0.5)
           + theme_classic()
-          + themes.theme(plot_title=themes.element_text(size=12)))
+          + themes.theme(plot_title=element_text(size=12)))
 
     # create data frame with information about linear regression and correlation for each Site
     lm_summary = collections.defaultdict(list)

create_plots/helpers_plot_ref_sim_comparisons.py

@@ -5,7 +5,7 @@
 
 from plotnine import ggplot, aes, geom_bar, scale_fill_brewer, facet_grid, geom_line, geom_point, geom_errorbar, \
     theme_bw, xlab, ylab, scale_color_manual, scale_fill_manual, coord_fixed, geom_abline, theme_classic, themes, \
-    facet_wrap, scale_shape_manual, scale_size_manual, scale_x_log10, ggtitle, labs, position_dodge
+    facet_wrap, scale_shape_manual, scale_size_manual, scale_x_log10, ggtitle, labs, position_dodge, element_text
 import numpy as np
 import pandas as pd
 from scipy.stats import beta
@@ -58,7 +58,7 @@ def compare_benchmark(combined_df):
           + coord_fixed(ratio=1, xlim=(min_value, max_value), ylim=(min_value, max_value))
           + geom_abline(slope=1, intercept=0, color='grey', alpha=0.5)
           + theme_classic()
-          + themes.theme(plot_title=themes.element_text(size=12)))
+          + themes.theme(plot_title=element_text(size=12)))
     return gg
 
 
@@ -249,7 +249,7 @@ def plot_par_dens_ref_sim_comparison(combined_df):
                + theme_bw()
                + ylab('fraction of population')
                + xlab('parasite density bin')
-               + themes.theme(axis_text_x=themes.element_text(angle=45))
+               + themes.theme(axis_text_x=element_text(angle=45))
                + ggtitle(cur_site)
                + scale_color_manual(values=color_manual)
                + scale_shape_manual(values=shape_manual)
@@ -742,6 +742,6 @@ def plot_infection_duration_dist_by_age(ref_df, sim_data, pos_thresh_dens, age_b
           + scale_fill_manual(values=color_manual_2)
           + labs(title='infection duration', x='infection duration (days)')
           # ylim(NA,max_y) +
-          + facet_grid(scales='fixed', facets=['censor_type', 'age_group']))
+          + facet_grid('censor_type', 'age_group', scales='fixed'))
 
     return gg

idmtools.ini

@@ -11,7 +11,7 @@ max_local_sims = 6
 
 [SLURM]
 type = COMPS
-endpoint = https://comps.idmod.org
+endpoint = https://comps2.idmod.org
 environment = Calculon
 priority = Normal
 simulation_root = $COMPS_PATH(USER)\output

requirements.txt

@@ -7,3 +7,4 @@ scipy~=1.9
 snakemake~=7.19
 plotnine~=0.10
 datar~=0.9
+pulp==2.7.0

setup.py

@@ -41,7 +41,7 @@ def load_arguments_from_file(filepath, arguments_list):
 
 setuptools.setup(
     name="malaria_model_validation",
-    version="1.2.0",
+    version="1.3.0",
     author=[author[0] for author in authors],
     author_email=[author[1] for author in authors],
     description="malaria-model_validation package",

simulations/add_suite.py

@@ -19,7 +19,7 @@ def add_suite(sites: list, suite_name: str = 'Malaria Model Validation Suite') -
     Returns:
         suite_id
     """
-    platform = Platform(manifest.platform_name)
+    platform = Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment)
     first_exp_found = True
     for site in sites:
         exp_id_file = get_comps_id_filename(site, level=0)

simulations/analyzers/AnnualSummaryReportAnalyzer.py

@@ -14,7 +14,6 @@
 from idmtools.core import ItemType
 from idmtools.core.platform_factory import Platform
 
-
 mpl.use('Agg')
 
 
@@ -78,19 +77,20 @@ def reduce(self, all_data: Dict[Union[IWorkflowItem, Simulation], Any]) -> Any:
 
         groupby_tags = self.sweep_variables
         groupby_tags.remove('Run_Number')
-        df_summarized = df_final.groupby(['Age']+groupby_tags)['Prevalence', 'Incidence'].apply(np.mean).reset_index()
-        df_summarized_std = df_final.groupby(['Age']+groupby_tags)['Prevalence', 'Incidence'].apply(np.std)
+        df_summarized = df_final.groupby(['Age']+groupby_tags, as_index=False).agg({'Prevalence': np.mean, 'Incidence': np.mean})
+        df_summarized_std = df_final.groupby(['Age']+groupby_tags).agg({'Prevalence': np.mean, 'Incidence': np.mean})
         for c in ['Prevalence', 'Incidence']:
             df_summarized[c + '_std'] = list(df_summarized_std[c])
 
         df_summarized.to_csv(os.path.join(self.working_dir, self.expt_name, "inc_prev_data_final.csv"))
 
 
 if __name__ == '__main__':
+    import simulations.manifest as manifest
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
     logger = getLogger()
-    with Platform('CALCULON') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Initialize the analyser class with the path of the output csv file
         analyzers = [AnnualSummaryReportAnalyzer()]

simulations/analyzers/InfectiousnessByParDensAgeAnalyzer.py

@@ -69,14 +69,15 @@ def reduce(self, all_data):
 
 
 if __name__ == '__main__':
+    import simulations.manifest as manifest
     # Set the experiment id you want to analyze
     experiment_id = 'b7126585-30b6-ec11-a9f6-9440c9be2c51'
     end_year = 65
 
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
     logger = getLogger()
-    with Platform('CALCULON') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Initialize the analyser class with the path of the output csv file
         analyzers = [InfectiousnessByParDensAgeAnalyzer(expt_name='Sugungum_1970',

simulations/analyzers/InsetChartAnalyzer.py

@@ -14,7 +14,6 @@
 from idmtools.core import ItemType
 from idmtools.core.platform_factory import Platform
 
-
 mpl.use('Agg')
 
 
@@ -103,10 +102,11 @@ def reduce(self, all_data: Dict[Union[IWorkflowItem, Simulation], Any]) -> Any:
 
 
 if __name__ == '__main__':
+    import simulations.manifest as manifest
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
     logger = getLogger()
-    with Platform('CALCULON') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Initialize the analyser class with the path of the output csv file
         analyzers = [InsetChartAnalyzer()]

simulations/analyzers/MonthlySummaryReportAnalyzer.py

@@ -69,14 +69,15 @@ def reduce(self, all_data):
 
 
 if __name__ == '__main__':
+    import simulations.manifest as manifest
     # Set the experiment id you want to analyze
     experiment_id = 'b7126585-30b6-ec11-a9f6-9440c9be2c51'
     end_year = 65
 
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
     logger = getLogger()
-    with Platform('CALCULON') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Initialize the analyser class with the path of the output csv file
         analyzers = [MonthlySummaryReportAnalyzer(expt_name='Sugungum_1970',

simulations/analyzers/ParDensAgeAnalyzer.py

@@ -76,6 +76,7 @@ def reduce(self, all_data):
 
 
 if __name__ == '__main__':
+    import simulations.manifest as manifest
 
     # Set the experiment id you want to analyze
     experiment_id = 'b7126585-30b6-ec11-a9f6-9440c9be2c51'
@@ -84,7 +85,7 @@ def reduce(self, all_data):
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
     logger = getLogger()
-    with Platform('CALCULON') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Initialize the analyser class with the path of the output csv file
         analyzers = [ParDensAgeAnalyzer(expt_name='Sugungum_1970',

simulations/analyzers/PatientReportAnalyzer.py

@@ -89,12 +89,13 @@ def reduce(self, all_data):
     from idmtools.analysis.analyze_manager import AnalyzeManager
     from idmtools.core import ItemType
     from idmtools.core.platform_factory import Platform
+    import simulations.manifest as manifest
 
     use_ssmt = True
 
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
-    with Platform('Calculon') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Set the experiment you want to analyze
         experiments = {

simulations/analyzers/PatientReportAnalyzer_laterDays.py

@@ -86,10 +86,11 @@ def reduce(self, all_data):
     from idmtools.core import ItemType
     from idmtools.core.platform_factory import Platform
     from general.manifest import analyzed_ouptut_path
+    import simulations.manifest as manifest
 
     # Set the platform where you want to run your analysis
     # In this case we are running in BELEGOST since the Work Item we are analyzing was run on COMPS
-    with Platform('Calculon') as platform:
+    with Platform(manifest.platform_name, endpoint=manifest.endpoint, environment=manifest.environment) as platform:
 
         # Set the experiment you want to analyze
         experiments = {'test_infectDuration_Navrongo2': 'aaa3fc9f-787a-ec11-a9f3-9440c9be2c51'}

simulations/create_auth_token_args.py

@@ -2,7 +2,7 @@
 import getpass
 from COMPS import Client
 from COMPS.CredentialPrompt import CredentialPrompt
-
+import simulations.manifest as manifest
 __comps_client_version = 10
 
 
@@ -29,7 +29,7 @@ def prompt(self):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 
-    parser.add_argument('--comps_url', '-c', default='https://comps.idmod.org', help='comps url')
+    parser.add_argument('--comps_url', '-c', default=manifest.endpoint, help='comps url')
     parser.add_argument('--username', '-u', help='enter username')
     parser.add_argument('--password', '-p', help='enter password')
 
@@ -43,4 +43,4 @@ def prompt(self):
     print(f"Login to {compshost} and cache the credentials.")
 
     Client.login(compshost, StaticCredentialPrompt(comps_url=args.comps_url, username=args.username,
-                                                   password=p))
+                                                   password=p))