Age base hint SFTs (#32)

* Update requiements_2018.txt to the latest dev packages

* Add helper function for Age Base HINT test to configure the Typhoid experiment and a Base class for the test

* Add 4 SFTs to make sure the Age Based HINT TransmissionMatrix are targeting the correct Age Bins.

* Add some description for the SFTs and the test classes.

* Add SFTs for the values in the AgeBasedHINT matrix.

* removing sampling rate, and updating a parameter name that didn't make sense. Without the sampling rate, the test passes

* update Jenkinsfile to install from requirements.txt and remove requirements_2018.txt

* fix test_vax_rollout_coverage SFT to have a dynamic tolerance for vaccine coverage check

* Update requirements.txt with emod_api == 1.31.0.dev5

* Changed TestDemographics file to use uniform initial age to avoid unrealistically old people. Added symlink to run_all_sft_tests as optional tool for linux.

* add dtk_post_process_baseline.py for test_age_base_hint_values_baseline

* check in the right version of dtk_post_process_baseline.py

* Changed all 2s to all 1.75. Changed Run_Number to 0. Changed post-proc to use commented out code that does integer age bin boundaries not decimal.

---------

Co-authored-by: SEATTLE\svetlanati <[email protected]>
Co-authored-by: Jonathan Bloedow <[email protected]>

Jenkinsfile

@@ -49,7 +49,7 @@ podTemplate(
 			def curDate = sh(returnStdout: true, script: "date").trim()
 			echo "The current date is ${curDate}"
 			echo "I am installing emodpy-typhoid from github source code"
-			sh "pip3 install -r requirements_2018.txt --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
+			sh "pip3 install -r requirements.txt --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
 			sh "pip3 list"
 			sh "pip3 install -e ."
 			sh "pip3 list"

requirements.txt

@@ -1,3 +1,3 @@
 emodpy==1.22.0.dev3
-emod-api==1.31.0.dev1
-emod-typhoid==0.0.5
+emod_api==1.31.0.dev5
+emod-typhoid==0.0.7.dev3

tests/sft_tests/Assets/TestDemographics_pak_updated.json

@@ -20,9 +20,9 @@
                 "BirthRate": 0
             },
             "IndividualAttributes": {
-                "AgeDistributionFlag": 3,
-                "AgeDistribution1": 0.0001,
-                "AgeDistribution2": 0,
+                "AgeDistributionFlag": 1,
+                "AgeDistribution1": 0,
+                "AgeDistribution2": 36500,
                 "PrevalenceDistributionFlag": 0,
                 "PrevalenceDistribution1": 0,
                 "PrevalenceDistribution2": 0,
@@ -9713,4 +9713,4 @@
 
         }
     ]
-}
+}

tests/sft_tests/hint_tests/hint_test_helper.py

@@ -0,0 +1,195 @@
+import os
+import sys
+import shutil
+
+from idm_test.dtk_test.integration.integration_test import IntegrationTest
+import emod_api.interventions.common as comm
+from idmtools.core.platform_factory import Platform
+
+from emodpy.emod_task import EMODTask
+from idmtools.entities.experiment import Experiment
+from idm_test.dtk_test.integration import manifest
+
+sys.path.append('../')
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from helper import year_to_days, setup
+
+BASE_YEAR = 2005
+SIMULATION_DURATION_IN_YEARS = 5
+CAMP_START_YEAR = 2006
+
+current_dir = os.path.dirname(__file__)
+Age_Bin_Edges_In_Years=[0, 5, 20, 60, -1]
+num_age_bin = len(Age_Bin_Edges_In_Years) - 1
+
+
+def set_param_fn(config):
+    """
+    Update the config parameters from default values.
+    """
+    print("Setting params.")
+    config.parameters.Simulation_Type = "TYPHOID_SIM"
+    config.parameters.Simulation_Duration = SIMULATION_DURATION_IN_YEARS * 365.0
+    config.parameters.Base_Individual_Sample_Rate = 0.2
+    config.parameters.Run_Number = 0
+
+    config.parameters.Base_Year = BASE_YEAR
+    config.parameters.Inset_Chart_Reporting_Start_Year = 2005
+    config.parameters.Inset_Chart_Reporting_Stop_Year = 2020
+    config.parameters.Enable_Demographics_Reporting = 0
+    config.parameters.Report_Typhoid_ByAgeAndGender_Start_Year = 2005
+    config.parameters.Report_Typhoid_ByAgeAndGender_Stop_Year = 2020
+
+    config.parameters.Typhoid_3year_Susceptible_Fraction = 0
+    config.parameters.Typhoid_6month_Susceptible_Fraction = 0
+    config.parameters.Typhoid_6year_Susceptible_Fraction = 0
+    config.parameters.Typhoid_Acute_Infectiousness = 13435
+    config.parameters.Typhoid_Carrier_Probability = 0.108
+    config.parameters.Typhoid_Carrier_Removal_Year = 2500
+    config.parameters.Typhoid_Chronic_Relative_Infectiousness = 0.241
+    config.parameters.Typhoid_Contact_Exposure_Rate = 0.06918859049226553
+    config.parameters.Typhoid_Environmental_Exposure_Rate = 0.06169346985005757
+    config.parameters.Typhoid_Environmental_Cutoff_Days = 157.20690133538764
+    config.parameters.Typhoid_Environmental_Peak_Start = 355.0579483941714
+    config.parameters.Typhoid_Environmental_Ramp_Down_Duration = 112.30224910440123
+    config.parameters.Typhoid_Environmental_Ramp_Up_Duration = 39.540475369174146
+    config.parameters.Typhoid_Exposure_Lambda = 7.0
+    config.parameters.Typhoid_Prepatent_Relative_Infectiousness = 0.5
+    config.parameters.Typhoid_Protection_Per_Infection = 0.98
+    config.parameters.Typhoid_Subclinical_Relative_Infectiousness = 1
+    config.parameters.Typhoid_Symptomatic_Fraction = 0.07
+
+    config.parameters.Demographics_Filenames = ["TestDemographics_pak_updated.json"]
+    config.parameters.Enable_Property_Output = 0
+    config.parameters.Report_Event_Recorder_Events = ["VaccineDistributed", "NewInfectionEvent"]
+    config.parameters["Listed_Events"] = ["VaccineDistributed"]  # old school
+
+    config.parameters.Age_Initialization_Distribution_Type = "DISTRIBUTION_COMPLEX"
+    config.parameters.Death_Rate_Dependence = "NONDISEASE_MORTALITY_BY_YEAR_AND_AGE_FOR_EACH_GENDER"
+    config.parameters.Birth_Rate_Dependence = "INDIVIDUAL_PREGNANCIES_BY_AGE_AND_YEAR"
+    # when using 2018 binary
+    import emodpy_typhoid.config as config_utils
+    config_utils.cleanup_for_2018_mode(config)
+    return config
+
+
+def build_camp():
+    import emod_api.campaign as camp
+
+    print(f"Telling emod-api to use {manifest.schema_file} as schema.")
+    camp.set_schema(manifest.schema_file)
+    import emodpy_typhoid.interventions.outbreak as ob
+    ob_event = ob.add_outbreak_individual(start_day=1,
+                                          demographic_coverage=0.05,
+                                          repetitions=1,
+                                          timesteps_between_repetitions=30
+                                          )
+    camp.add(ob_event)
+
+    def add_historical_vax(camp, ria_coverage=0.75, camp_coverage=0.75, efficacy=0.8, expiration=3650):
+        import emodpy_typhoid.interventions.typhoid_vaccine as tv
+
+        ria = tv.new_routine_immunization(camp,
+                                          efficacy=efficacy,
+                                          constant_period=0,
+                                          expected_expiration=expiration,
+                                          # decay_constant=values['decay_constant'],
+                                          start_day=year_to_days(CAMP_START_YEAR),
+                                          coverage=ria_coverage)
+        tv_iv = tv.new_vax(camp,
+                           efficacy=efficacy,
+                           expected_expiration=expiration,
+                           # decay_constant=values['decay_constant'],
+                           constant_period=0)
+
+        notification_iv = comm.BroadcastEvent(camp, "VaccineDistributed")
+        camp.add(ria)
+
+        one_time_campaign = comm.ScheduledCampaignEvent(camp,
+                                                        Start_Day=year_to_days(CAMP_START_YEAR),
+                                                        Intervention_List=[tv_iv, notification_iv],
+                                                        Demographic_Coverage=camp_coverage,
+                                                        Target_Age_Min=0.75,
+                                                        Target_Age_Max=15
+                                                        )
+        camp.add(one_time_campaign)
+
+    # add_historical_vax(camp, ria_coverage=1.0, camp_coverage=1.0, efficacy=1.0, expiration=36500)
+    return camp
+
+
+def build_demog():
+    """
+    Build a demographics input file for the DTK using emod_api.
+    """
+    import emodpy_typhoid.demographics.TyphoidDemographics as Demographics  # OK to call into emod-api
+
+    demog = Demographics.from_template_node(lat=0, lon=0, pop=10000, name=1, forced_id=1)
+    return demog
+
+
+def build_demog_target_all_age_bin(value=1):
+    """
+    Build a demographics input file for the DTK using emod_api with AgeDependentTransmission matrix all set to a
+    constant value.
+    """
+    demog = build_demog()
+
+    demog.AddAgeDependentTransmission(
+        Age_Bin_Edges_In_Years=Age_Bin_Edges_In_Years.copy(),
+        TransmissionMatrix=[[value] * num_age_bin for _ in range(num_age_bin)]
+    )
+
+    return demog
+
+
+def build_demog_target_one_age_bin(group_index=0, value=1):
+    """
+    Build a demographics input file for the DTK using emod_api with AgeDependentTransmission matrix. The transmission
+    matrix will have a constant value for one group and 0 for the other groups
+    """
+    demog = build_demog()
+    TransmissionMatrix = [[0] * num_age_bin for _ in range(num_age_bin)]
+
+    for row in TransmissionMatrix:
+        row[group_index] = value
+
+    demog.AddAgeDependentTransmission(
+        Age_Bin_Edges_In_Years=Age_Bin_Edges_In_Years.copy(),
+        TransmissionMatrix=TransmissionMatrix
+    )
+
+    return demog
+
+
+class TestAgeBaseHint(IntegrationTest):
+    """
+    Base test class for Age Base HINT test, that inherits the IntegrationTest class from
+    dm_test.dtk_test.integration.integration_test.
+    Each new test class will call the age_base_hint_test() function to perform the test.
+    """
+    def setUp(self):
+        self.test_name = self.case_name = str(self.test_name) + "--" + self._testMethodName
+        self.platform = Platform("SLURM2", priority="Normal")
+        setup(self.platform)
+
+    def tearDown(self) -> None:
+        exp_folder = self.experiment.id
+        if os.path.exists(exp_folder) and os.path.isdir(exp_folder):
+            shutil.rmtree(exp_folder, ignore_errors=True)
+
+    def age_base_hint_test(self, custom_build_demog):
+        task = EMODTask.from_default2(config_path="config.json", eradication_path=manifest.eradication_path,
+                                      campaign_builder=build_camp, demog_builder=custom_build_demog,
+                                      schema_path=manifest.schema_file, param_custom_cb=set_param_fn,
+                                      ep4_custom_cb=self._add_ep4)
+
+        task.common_assets.add_directory(os.path.join("..", "..", "Assets"))
+        task.config.parameters.Demographics_Filenames = ["demographics.json", "TestDemographics_pak_updated.json"]
+        task.set_sif(manifest.sft_id)
+        self.experiment = Experiment.from_task(task, name=self.test_name)
+        # The last step is to call run() on the ExperimentManager to run the simulations.
+        self.experiment.run(wait_until_done=True)
+        task.handle_experiment_completion(self.experiment)
+        self.experiment = self.experiment
+        self._check_result()

tests/sft_tests/hint_tests/run_all_sft_tests.py

@@ -0,0 +1 @@
+../run_all_sft_tests.py

tests/sft_tests/hint_tests/test_age_base_hint_target/ep4_dir/dtk_post_process.py

@@ -0,0 +1,119 @@
+#!/usr/bin/python
+import os.path
+import json
+import pandas as pd
+
+from idm_test.dtk_test.sft_class import arg_parser, SFT
+
+
+class AgeBaseHINTTargetTest(SFT):
+    """
+    SFTs that testing the targeting Age Bin in the Age Bin HINT TransmissionMatrix.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_age_base_hint_matrix(self):
+        """
+        load the demographics json and get the age base HINT related data
+        """
+        with open('Assets/demographics.json', 'r') as f:
+            demog = json.load(f)
+            age_bin_edges_in_years = demog['Defaults']['IndividualProperties'][0]['Age_Bin_Edges_In_Years']
+            transmission_matrix = demog['Defaults']['IndividualProperties'][0]['TransmissionMatrix']
+            contact = transmission_matrix['contact']['Matrix']
+            environmental = transmission_matrix['environmental']['Matrix']
+            return age_bin_edges_in_years, contact, environmental
+
+    # overwrite the test method
+    def test(self):
+        self.success = True
+
+        # Load Age Base HINT parameters from Demographics file
+        age_bin_edges_in_years, contact, environmental = self.get_age_base_hint_matrix()
+
+        # Replace the last age bin edge -1 with a large year of age. It will be used to generate the IntervalIndex for
+        # dataframe later.
+        age_bin_edges_in_years[-1] = 150
+
+        with open(self.report_name, "w") as outfile:
+            if contact != environmental:
+                self.success = False
+                outfile.write(
+                    f"BAD: Expected the same transmission matix for both contact and environmental routes. Found:"
+                    f" contact = {contact}, environmental = {environmental}.\n")
+            else:
+                transmission_matrix = contact
+                found_group = None
+                for idx, x in enumerate(transmission_matrix[0]):
+                    if x != 0:
+                        found_group = idx
+                        break
+                if found_group is None:
+                    self.success = False
+                    outfile.write(
+                        f"BAD: Expected some non zero value in TransmissionMatrix. Please check TransmissionMatrix = "
+                        f"{transmission_matrix}\n")
+                else:
+                    with open(os.path.join(self.output_folder, "ReportEventRecorder.csv"), 'r') as infile:
+                        df = pd.read_csv(infile)
+                        df.columns = df.columns.to_series().apply(lambda x: x.strip())
+                        # Filter the dataframe for NewInfectionEvent
+                        new_infected_df = df.loc[(df['Event_Name'] == 'NewInfectionEvent')]
+                        # Add a new 'Age_Year' column to the DataFrame by converting 'Age' to years.
+                        new_infected_df['Age_Year'] = new_infected_df['Age'] / 365.0
+                        # Add a new 'Age_Bin' column to the DataFrame based on age bin edges.
+                        bins = pd.IntervalIndex.from_breaks(age_bin_edges_in_years)
+                        new_infected_df['Age_Bin'] = pd.cut(new_infected_df['Age_Year'], bins=bins, right=False)
+
+                        # Count NewInfectionEvent by timestamp and Age_Bin
+                        count_df = new_infected_df.groupby(['Year', 'Age_Bin']).size().reset_index(name='Count')
+                        count_df.reset_index(drop=True, inplace=True)
+
+                        # Save dataframes to csv for debugging
+                        new_infected_df.to_csv("ReportEventRecorder_AgeBin.csv")
+                        count_df.to_csv("ReportEventRecorder_AgeBin_Count.csv")
+
+                        # ignore the first timestep where we have the outbreak
+                        labels = pd.unique(count_df['Age_Bin']).tolist()
+                        count_df = count_df.loc[(count_df['Year'] != 2005)]
+                        outfile.write(f"Age Bins: {labels}.\n")
+                        left = age_bin_edges_in_years[found_group]
+                        right = age_bin_edges_in_years[found_group+1]
+                        expected_age_bin_interval = pd.Interval(left=left, right=right, closed='right')
+
+                        for label in labels:
+                            if label != expected_age_bin_interval:
+                                if count_df.loc[count_df['Age_Bin'] == label]['Count'].sum() != 0:
+                                    self.success = False
+                                    outfile.write(f"BAD: Expected no transmission for age bin: {label}, found "
+                                                  f"{count_df.loc[count_df['Age_Bin'] == label]['Count'].sum()} new infections.\n")
+                                else:
+                                    outfile.write(f"Good: There is no transmission for age bin: {label}.\n")
+                            else:
+                                new_infection_total = count_df.loc[count_df['Age_Bin'] == label]['Count'].sum()
+                                if new_infection_total == 0:
+                                    self.success = False
+                                    outfile.write(f"BAD: Expected some transmissions for age bin: {label}, found 0 "
+                                                  f"new infection.\n")
+                                else:
+                                    outfile.write(f"Good: There are some transmission for age bin: {label},found "
+                                                  f"{new_infection_total} new infection.\n")
+
+        return self.success
+
+
+def application(output_folder="output", my_arg=None):
+    if not my_arg:
+        my_sft = AgeBaseHINTTargetTest(stdout='stdout.txt')
+    else:
+        my_sft = AgeBaseHINTTargetTest(
+            output=my_arg.output, stdout='stdout.txt', json_report=my_arg.json_report, event_csv=my_arg.event_csv,
+            config=my_arg.config, campaign=my_arg.campaign, report_name=my_arg.report_name, debug=my_arg.debug)
+    my_sft.run()
+
+
+if __name__ == "__main__":
+    # execute only if run as a script
+    my_arg = arg_parser()
+    application(my_arg=my_arg)

tests/sft_tests/hint_tests/test_age_base_hint_target/test_age_base_hint_target.py

@@ -0,0 +1,40 @@
+import os
+import sys
+from functools import partial
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append('../')
+from hint_test_helper import TestAgeBaseHint, build_camp, build_demog_target_one_age_bin, set_param_fn
+
+
+class TestAgeBaseHintTargetOneAgeBin(TestAgeBaseHint):
+    """
+    A test class that submit SFTs to Comps and run the dtk_post_process.py in ep4_dir folder and write the SFT result
+    as a tag of for the simulation in Comps.
+    Each test case will define a build_demog function and call the main test function in the base class as
+    self.age_base_hint_test(build_demog).
+    """
+    @classmethod
+    def setUpClass(cls):
+        cls.test_name = os.path.basename(__file__)
+
+    def test_age_base_hint_targe_first_age_bin(self):
+        build_demog = partial(build_demog_target_one_age_bin, group_index=0, value=1)
+        self.age_base_hint_test(build_demog)
+
+    def test_age_base_hint_targe_second_age_bin(self):
+        build_demog = partial(build_demog_target_one_age_bin, group_index=1, value=0.5)
+        self.age_base_hint_test(build_demog)
+
+    def test_age_base_hint_targe_third_age_bin(self):
+        build_demog = partial(build_demog_target_one_age_bin, group_index=2, value=0.8)
+        self.age_base_hint_test(build_demog)
+
+    def test_age_base_hint_targe_last_age_bin(self):
+        build_demog = partial(build_demog_target_one_age_bin, group_index=3, value=2)
+        self.age_base_hint_test(build_demog)
+
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()