From 99a597aecad8b54a6b4fb73a817f913642ce277c Mon Sep 17 00:00:00 2001 From: YeChen-IDM Date: Tue, 23 Jan 2024 12:27:58 -0800 Subject: [PATCH] Age base hint SFTs (#32) * Update requiements_2018.txt to the latest dev packages * Add helper function for Age Base HINT test to configure the Typhoid experiment and a Base class for the test * Add 4 SFTs to make sure the Age Based HINT TransmissionMatrix are targeting the correct Age Bins. * Add some description for the SFTs and the test classes. * Add SFTs for the values in the AgeBasedHINT matrix. * removing sampling rate, and updating a parameter name that didn't make sense. Without the sampling rate, the test passes * update Jenkinsfile to install from requirements.txt and remove requirements_2018.txt * fix test_vax_rollout_coverage SFT to have a dynamic tolerance for vaccine coverage check * Update requirements.txt with emod_api == 1.31.0.dev5 * Changed TestDemographics file to use uniform initial age to avoid unrealistically old people. Added symlink to run_all_sft_tests as optional tool for linux. * add dtk_post_process_baseline.py for test_age_base_hint_values_baseline * check in the right version of dtk_post_process_baseline.py * Changed all 2s to all 1.75. Changed Run_Number to 0. Changed post-proc to use commented out code that does integer age bin boundaries not decimal. --------- Co-authored-by: SEATTLE\svetlanati Co-authored-by: Jonathan Bloedow --- Jenkinsfile | 2 +- requirements.txt | 4 +- .../Assets/TestDemographics_pak_updated.json | 8 +- .../sft_tests/hint_tests/hint_test_helper.py | 195 ++++++++++++++++++ .../sft_tests/hint_tests/run_all_sft_tests.py | 1 + .../ep4_dir/dtk_post_process.py | 119 +++++++++++ .../test_age_base_hint_target.py | 40 ++++ .../ep4_dir/dtk_post_process.py | 135 ++++++++++++ .../ep4_dir/dtk_post_process_baseline.py | 82 ++++++++ .../test_age_base_hint_values.py | 67 ++++++ .../ep4_dir/dtk_post_process.py | 18 +- tests/workflow_tests/test_multiroute_HINT.py | 10 +- 12 files changed, 666 insertions(+), 15 deletions(-) create mode 100644 tests/sft_tests/hint_tests/hint_test_helper.py create mode 120000 tests/sft_tests/hint_tests/run_all_sft_tests.py create mode 100644 tests/sft_tests/hint_tests/test_age_base_hint_target/ep4_dir/dtk_post_process.py create mode 100644 tests/sft_tests/hint_tests/test_age_base_hint_target/test_age_base_hint_target.py create mode 100644 tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process.py create mode 100644 tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process_baseline.py create mode 100644 tests/sft_tests/hint_tests/test_age_base_hint_values/test_age_base_hint_values.py diff --git a/Jenkinsfile b/Jenkinsfile index 7570ecd..43f3a3f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,7 +49,7 @@ podTemplate( def curDate = sh(returnStdout: true, script: "date").trim() echo "The current date is ${curDate}" echo "I am installing emodpy-typhoid from github source code" - sh "pip3 install -r requirements_2018.txt --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple" + sh "pip3 install -r requirements.txt --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple" sh "pip3 list" sh "pip3 install -e ." sh "pip3 list" diff --git a/requirements.txt b/requirements.txt index 3946a35..4609c6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ emodpy==1.22.0.dev3 -emod-api==1.31.0.dev1 -emod-typhoid==0.0.5 +emod_api==1.31.0.dev5 +emod-typhoid==0.0.7.dev3 diff --git a/tests/sft_tests/Assets/TestDemographics_pak_updated.json b/tests/sft_tests/Assets/TestDemographics_pak_updated.json index 588bfdf..0897706 100644 --- a/tests/sft_tests/Assets/TestDemographics_pak_updated.json +++ b/tests/sft_tests/Assets/TestDemographics_pak_updated.json @@ -20,9 +20,9 @@ "BirthRate": 0 }, "IndividualAttributes": { - "AgeDistributionFlag": 3, - "AgeDistribution1": 0.0001, - "AgeDistribution2": 0, + "AgeDistributionFlag": 1, + "AgeDistribution1": 0, + "AgeDistribution2": 36500, "PrevalenceDistributionFlag": 0, "PrevalenceDistribution1": 0, "PrevalenceDistribution2": 0, @@ -9713,4 +9713,4 @@ } ] -} \ No newline at end of file +} diff --git a/tests/sft_tests/hint_tests/hint_test_helper.py b/tests/sft_tests/hint_tests/hint_test_helper.py new file mode 100644 index 0000000..6d65dea --- /dev/null +++ b/tests/sft_tests/hint_tests/hint_test_helper.py @@ -0,0 +1,195 @@ +import os +import sys +import shutil + +from idm_test.dtk_test.integration.integration_test import IntegrationTest +import emod_api.interventions.common as comm +from idmtools.core.platform_factory import Platform + +from emodpy.emod_task import EMODTask +from idmtools.entities.experiment import Experiment +from idm_test.dtk_test.integration import manifest + +sys.path.append('../') +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import year_to_days, setup + +BASE_YEAR = 2005 +SIMULATION_DURATION_IN_YEARS = 5 +CAMP_START_YEAR = 2006 + +current_dir = os.path.dirname(__file__) +Age_Bin_Edges_In_Years=[0, 5, 20, 60, -1] +num_age_bin = len(Age_Bin_Edges_In_Years) - 1 + + +def set_param_fn(config): + """ + Update the config parameters from default values. + """ + print("Setting params.") + config.parameters.Simulation_Type = "TYPHOID_SIM" + config.parameters.Simulation_Duration = SIMULATION_DURATION_IN_YEARS * 365.0 + config.parameters.Base_Individual_Sample_Rate = 0.2 + config.parameters.Run_Number = 0 + + config.parameters.Base_Year = BASE_YEAR + config.parameters.Inset_Chart_Reporting_Start_Year = 2005 + config.parameters.Inset_Chart_Reporting_Stop_Year = 2020 + config.parameters.Enable_Demographics_Reporting = 0 + config.parameters.Report_Typhoid_ByAgeAndGender_Start_Year = 2005 + config.parameters.Report_Typhoid_ByAgeAndGender_Stop_Year = 2020 + + config.parameters.Typhoid_3year_Susceptible_Fraction = 0 + config.parameters.Typhoid_6month_Susceptible_Fraction = 0 + config.parameters.Typhoid_6year_Susceptible_Fraction = 0 + config.parameters.Typhoid_Acute_Infectiousness = 13435 + config.parameters.Typhoid_Carrier_Probability = 0.108 + config.parameters.Typhoid_Carrier_Removal_Year = 2500 + config.parameters.Typhoid_Chronic_Relative_Infectiousness = 0.241 + config.parameters.Typhoid_Contact_Exposure_Rate = 0.06918859049226553 + config.parameters.Typhoid_Environmental_Exposure_Rate = 0.06169346985005757 + config.parameters.Typhoid_Environmental_Cutoff_Days = 157.20690133538764 + config.parameters.Typhoid_Environmental_Peak_Start = 355.0579483941714 + config.parameters.Typhoid_Environmental_Ramp_Down_Duration = 112.30224910440123 + config.parameters.Typhoid_Environmental_Ramp_Up_Duration = 39.540475369174146 + config.parameters.Typhoid_Exposure_Lambda = 7.0 + config.parameters.Typhoid_Prepatent_Relative_Infectiousness = 0.5 + config.parameters.Typhoid_Protection_Per_Infection = 0.98 + config.parameters.Typhoid_Subclinical_Relative_Infectiousness = 1 + config.parameters.Typhoid_Symptomatic_Fraction = 0.07 + + config.parameters.Demographics_Filenames = ["TestDemographics_pak_updated.json"] + config.parameters.Enable_Property_Output = 0 + config.parameters.Report_Event_Recorder_Events = ["VaccineDistributed", "NewInfectionEvent"] + config.parameters["Listed_Events"] = ["VaccineDistributed"] # old school + + config.parameters.Age_Initialization_Distribution_Type = "DISTRIBUTION_COMPLEX" + config.parameters.Death_Rate_Dependence = "NONDISEASE_MORTALITY_BY_YEAR_AND_AGE_FOR_EACH_GENDER" + config.parameters.Birth_Rate_Dependence = "INDIVIDUAL_PREGNANCIES_BY_AGE_AND_YEAR" + # when using 2018 binary + import emodpy_typhoid.config as config_utils + config_utils.cleanup_for_2018_mode(config) + return config + + +def build_camp(): + import emod_api.campaign as camp + + print(f"Telling emod-api to use {manifest.schema_file} as schema.") + camp.set_schema(manifest.schema_file) + import emodpy_typhoid.interventions.outbreak as ob + ob_event = ob.add_outbreak_individual(start_day=1, + demographic_coverage=0.05, + repetitions=1, + timesteps_between_repetitions=30 + ) + camp.add(ob_event) + + def add_historical_vax(camp, ria_coverage=0.75, camp_coverage=0.75, efficacy=0.8, expiration=3650): + import emodpy_typhoid.interventions.typhoid_vaccine as tv + + ria = tv.new_routine_immunization(camp, + efficacy=efficacy, + constant_period=0, + expected_expiration=expiration, + # decay_constant=values['decay_constant'], + start_day=year_to_days(CAMP_START_YEAR), + coverage=ria_coverage) + tv_iv = tv.new_vax(camp, + efficacy=efficacy, + expected_expiration=expiration, + # decay_constant=values['decay_constant'], + constant_period=0) + + notification_iv = comm.BroadcastEvent(camp, "VaccineDistributed") + camp.add(ria) + + one_time_campaign = comm.ScheduledCampaignEvent(camp, + Start_Day=year_to_days(CAMP_START_YEAR), + Intervention_List=[tv_iv, notification_iv], + Demographic_Coverage=camp_coverage, + Target_Age_Min=0.75, + Target_Age_Max=15 + ) + camp.add(one_time_campaign) + + # add_historical_vax(camp, ria_coverage=1.0, camp_coverage=1.0, efficacy=1.0, expiration=36500) + return camp + + +def build_demog(): + """ + Build a demographics input file for the DTK using emod_api. + """ + import emodpy_typhoid.demographics.TyphoidDemographics as Demographics # OK to call into emod-api + + demog = Demographics.from_template_node(lat=0, lon=0, pop=10000, name=1, forced_id=1) + return demog + + +def build_demog_target_all_age_bin(value=1): + """ + Build a demographics input file for the DTK using emod_api with AgeDependentTransmission matrix all set to a + constant value. + """ + demog = build_demog() + + demog.AddAgeDependentTransmission( + Age_Bin_Edges_In_Years=Age_Bin_Edges_In_Years.copy(), + TransmissionMatrix=[[value] * num_age_bin for _ in range(num_age_bin)] + ) + + return demog + + +def build_demog_target_one_age_bin(group_index=0, value=1): + """ + Build a demographics input file for the DTK using emod_api with AgeDependentTransmission matrix. The transmission + matrix will have a constant value for one group and 0 for the other groups + """ + demog = build_demog() + TransmissionMatrix = [[0] * num_age_bin for _ in range(num_age_bin)] + + for row in TransmissionMatrix: + row[group_index] = value + + demog.AddAgeDependentTransmission( + Age_Bin_Edges_In_Years=Age_Bin_Edges_In_Years.copy(), + TransmissionMatrix=TransmissionMatrix + ) + + return demog + + +class TestAgeBaseHint(IntegrationTest): + """ + Base test class for Age Base HINT test, that inherits the IntegrationTest class from + dm_test.dtk_test.integration.integration_test. + Each new test class will call the age_base_hint_test() function to perform the test. + """ + def setUp(self): + self.test_name = self.case_name = str(self.test_name) + "--" + self._testMethodName + self.platform = Platform("SLURM2", priority="Normal") + setup(self.platform) + + def tearDown(self) -> None: + exp_folder = self.experiment.id + if os.path.exists(exp_folder) and os.path.isdir(exp_folder): + shutil.rmtree(exp_folder, ignore_errors=True) + + def age_base_hint_test(self, custom_build_demog): + task = EMODTask.from_default2(config_path="config.json", eradication_path=manifest.eradication_path, + campaign_builder=build_camp, demog_builder=custom_build_demog, + schema_path=manifest.schema_file, param_custom_cb=set_param_fn, + ep4_custom_cb=self._add_ep4) + + task.common_assets.add_directory(os.path.join("..", "..", "Assets")) + task.config.parameters.Demographics_Filenames = ["demographics.json", "TestDemographics_pak_updated.json"] + task.set_sif(manifest.sft_id) + self.experiment = Experiment.from_task(task, name=self.test_name) + # The last step is to call run() on the ExperimentManager to run the simulations. + self.experiment.run(wait_until_done=True) + task.handle_experiment_completion(self.experiment) + self.experiment = self.experiment + self._check_result() diff --git a/tests/sft_tests/hint_tests/run_all_sft_tests.py b/tests/sft_tests/hint_tests/run_all_sft_tests.py new file mode 120000 index 0000000..9e125eb --- /dev/null +++ b/tests/sft_tests/hint_tests/run_all_sft_tests.py @@ -0,0 +1 @@ +../run_all_sft_tests.py \ No newline at end of file diff --git a/tests/sft_tests/hint_tests/test_age_base_hint_target/ep4_dir/dtk_post_process.py b/tests/sft_tests/hint_tests/test_age_base_hint_target/ep4_dir/dtk_post_process.py new file mode 100644 index 0000000..5f5c537 --- /dev/null +++ b/tests/sft_tests/hint_tests/test_age_base_hint_target/ep4_dir/dtk_post_process.py @@ -0,0 +1,119 @@ +#!/usr/bin/python +import os.path +import json +import pandas as pd + +from idm_test.dtk_test.sft_class import arg_parser, SFT + + +class AgeBaseHINTTargetTest(SFT): + """ + SFTs that testing the targeting Age Bin in the Age Bin HINT TransmissionMatrix. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def get_age_base_hint_matrix(self): + """ + load the demographics json and get the age base HINT related data + """ + with open('Assets/demographics.json', 'r') as f: + demog = json.load(f) + age_bin_edges_in_years = demog['Defaults']['IndividualProperties'][0]['Age_Bin_Edges_In_Years'] + transmission_matrix = demog['Defaults']['IndividualProperties'][0]['TransmissionMatrix'] + contact = transmission_matrix['contact']['Matrix'] + environmental = transmission_matrix['environmental']['Matrix'] + return age_bin_edges_in_years, contact, environmental + + # overwrite the test method + def test(self): + self.success = True + + # Load Age Base HINT parameters from Demographics file + age_bin_edges_in_years, contact, environmental = self.get_age_base_hint_matrix() + + # Replace the last age bin edge -1 with a large year of age. It will be used to generate the IntervalIndex for + # dataframe later. + age_bin_edges_in_years[-1] = 150 + + with open(self.report_name, "w") as outfile: + if contact != environmental: + self.success = False + outfile.write( + f"BAD: Expected the same transmission matix for both contact and environmental routes. Found:" + f" contact = {contact}, environmental = {environmental}.\n") + else: + transmission_matrix = contact + found_group = None + for idx, x in enumerate(transmission_matrix[0]): + if x != 0: + found_group = idx + break + if found_group is None: + self.success = False + outfile.write( + f"BAD: Expected some non zero value in TransmissionMatrix. Please check TransmissionMatrix = " + f"{transmission_matrix}\n") + else: + with open(os.path.join(self.output_folder, "ReportEventRecorder.csv"), 'r') as infile: + df = pd.read_csv(infile) + df.columns = df.columns.to_series().apply(lambda x: x.strip()) + # Filter the dataframe for NewInfectionEvent + new_infected_df = df.loc[(df['Event_Name'] == 'NewInfectionEvent')] + # Add a new 'Age_Year' column to the DataFrame by converting 'Age' to years. + new_infected_df['Age_Year'] = new_infected_df['Age'] / 365.0 + # Add a new 'Age_Bin' column to the DataFrame based on age bin edges. + bins = pd.IntervalIndex.from_breaks(age_bin_edges_in_years) + new_infected_df['Age_Bin'] = pd.cut(new_infected_df['Age_Year'], bins=bins, right=False) + + # Count NewInfectionEvent by timestamp and Age_Bin + count_df = new_infected_df.groupby(['Year', 'Age_Bin']).size().reset_index(name='Count') + count_df.reset_index(drop=True, inplace=True) + + # Save dataframes to csv for debugging + new_infected_df.to_csv("ReportEventRecorder_AgeBin.csv") + count_df.to_csv("ReportEventRecorder_AgeBin_Count.csv") + + # ignore the first timestep where we have the outbreak + labels = pd.unique(count_df['Age_Bin']).tolist() + count_df = count_df.loc[(count_df['Year'] != 2005)] + outfile.write(f"Age Bins: {labels}.\n") + left = age_bin_edges_in_years[found_group] + right = age_bin_edges_in_years[found_group+1] + expected_age_bin_interval = pd.Interval(left=left, right=right, closed='right') + + for label in labels: + if label != expected_age_bin_interval: + if count_df.loc[count_df['Age_Bin'] == label]['Count'].sum() != 0: + self.success = False + outfile.write(f"BAD: Expected no transmission for age bin: {label}, found " + f"{count_df.loc[count_df['Age_Bin'] == label]['Count'].sum()} new infections.\n") + else: + outfile.write(f"Good: There is no transmission for age bin: {label}.\n") + else: + new_infection_total = count_df.loc[count_df['Age_Bin'] == label]['Count'].sum() + if new_infection_total == 0: + self.success = False + outfile.write(f"BAD: Expected some transmissions for age bin: {label}, found 0 " + f"new infection.\n") + else: + outfile.write(f"Good: There are some transmission for age bin: {label},found " + f"{new_infection_total} new infection.\n") + + return self.success + + +def application(output_folder="output", my_arg=None): + if not my_arg: + my_sft = AgeBaseHINTTargetTest(stdout='stdout.txt') + else: + my_sft = AgeBaseHINTTargetTest( + output=my_arg.output, stdout='stdout.txt', json_report=my_arg.json_report, event_csv=my_arg.event_csv, + config=my_arg.config, campaign=my_arg.campaign, report_name=my_arg.report_name, debug=my_arg.debug) + my_sft.run() + + +if __name__ == "__main__": + # execute only if run as a script + my_arg = arg_parser() + application(my_arg=my_arg) diff --git a/tests/sft_tests/hint_tests/test_age_base_hint_target/test_age_base_hint_target.py b/tests/sft_tests/hint_tests/test_age_base_hint_target/test_age_base_hint_target.py new file mode 100644 index 0000000..f149813 --- /dev/null +++ b/tests/sft_tests/hint_tests/test_age_base_hint_target/test_age_base_hint_target.py @@ -0,0 +1,40 @@ +import os +import sys +from functools import partial + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append('../') +from hint_test_helper import TestAgeBaseHint, build_camp, build_demog_target_one_age_bin, set_param_fn + + +class TestAgeBaseHintTargetOneAgeBin(TestAgeBaseHint): + """ + A test class that submit SFTs to Comps and run the dtk_post_process.py in ep4_dir folder and write the SFT result + as a tag of for the simulation in Comps. + Each test case will define a build_demog function and call the main test function in the base class as + self.age_base_hint_test(build_demog). + """ + @classmethod + def setUpClass(cls): + cls.test_name = os.path.basename(__file__) + + def test_age_base_hint_targe_first_age_bin(self): + build_demog = partial(build_demog_target_one_age_bin, group_index=0, value=1) + self.age_base_hint_test(build_demog) + + def test_age_base_hint_targe_second_age_bin(self): + build_demog = partial(build_demog_target_one_age_bin, group_index=1, value=0.5) + self.age_base_hint_test(build_demog) + + def test_age_base_hint_targe_third_age_bin(self): + build_demog = partial(build_demog_target_one_age_bin, group_index=2, value=0.8) + self.age_base_hint_test(build_demog) + + def test_age_base_hint_targe_last_age_bin(self): + build_demog = partial(build_demog_target_one_age_bin, group_index=3, value=2) + self.age_base_hint_test(build_demog) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process.py b/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process.py new file mode 100644 index 0000000..0699bff --- /dev/null +++ b/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process.py @@ -0,0 +1,135 @@ +#!/usr/bin/python +import os.path +import json +import pandas as pd +import matplotlib.pyplot as plt + +from idm_test.dtk_test.sft_class import arg_parser, SFT + + +class AgeBaseHINTValueTest(SFT): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def get_age_base_hint_matrix(self): + """ + load the demographics json and get the age base HINT related data + """ + with open('Assets/demographics.json', 'r') as f: + demog = json.load(f) + age_bin_edges_in_years = demog['Defaults']['IndividualProperties'][0]['Age_Bin_Edges_In_Years'] + transmission_matrix = demog['Defaults']['IndividualProperties'][0]['TransmissionMatrix'] + contact = transmission_matrix['contact']['Matrix'] + environmental = transmission_matrix['environmental']['Matrix'] + return age_bin_edges_in_years, contact, environmental + + # overwrite the test method + def test(self): + self.success = True + + # Load Age Base HINT parameters from Demographics file + age_bin_edges_in_years, contact, environmental = self.get_age_base_hint_matrix() + + # Replace the last age bin edge -1 with a large year of age. It will be used to generate the IntervalIndex for + # dataframe later. + age_bin_edges_in_years[-1] = 125 + + with open(self.report_name, "w") as outfile: + if contact != environmental: + self.success = False + outfile.write( + f"BAD: Expected the same transmission matix for both contact and environmental routes. Found:" + f" contact = {contact}, environmental = {environmental}.\n") + else: + transmission_matrix = contact + for transmission_rate in transmission_matrix: + for idx, x in enumerate(transmission_matrix[0]): + if transmission_rate[idx] != x: + self.success = False + outfile.write( + f"BAD: Expected the same transmission rate for the same age bin. please check the " + f"TransmissionMatrix.\n") + break + if self.success: + with open(os.path.join(self.output_folder, "ReportEventRecorder.csv"), 'r') as infile: + df = pd.read_csv(infile) + df.columns = df.columns.to_series().apply(lambda x: x.strip()) + # Filter the dataframe for NewInfectionEvent + new_infected_df = df.loc[(df['Event_Name'] == 'NewInfectionEvent')] + # Add a new 'Age_Year' column to the DataFrame by converting 'Age' to years. + new_infected_df['Age_Year'] = new_infected_df['Age'] / 365.0 + # Add a new 'Age_Bin' column to the DataFrame based on age bin edges. + bins = pd.IntervalIndex.from_breaks(age_bin_edges_in_years) + new_infected_df['Age_Bin'] = pd.cut(new_infected_df['Age_Year'], bins=bins, right=False).astype(str) + + # Save dataframes to csv for debugging + new_infected_df.to_csv("ReportEventRecorder_AgeBin.csv") + + # ignore the first timestep where we have the outbreak + new_infected_df = new_infected_df.loc[(new_infected_df['Year'] != 2005)] + + # Count NewInfectionEvent by Age_Bin + count_dict = {age_bin: count for age_bin, count in new_infected_df.groupby('Age_Bin').size().items()} + + # Plot the dictionary into histogram plot for debugging + if len(count_dict): + self.plot_bar(count_dict) + outfile.write(f'{count_dict}\n') + + for idx, rate in enumerate(transmission_matrix[0]): + left = age_bin_edges_in_years[idx] + right = age_bin_edges_in_years[idx + 1] + age_bin_interval = str(pd.Interval(left=left, right=right, closed='right')) + # workaround for old Pandas library + #age_bin_interval = f'({left:.1f}, {right:.1f}]' + if rate == 0: + if age_bin_interval in count_dict: + self.success = False + outfile.write( + f"BAD: Expected no transmission in age bin {age_bin_interval} since " + f"transmission rate is 0.\n") + else: + outfile.write( + f"GOOD: There is no transmission in age bin {age_bin_interval} since " + f"transmission rate is 0.\n") + else: + if age_bin_interval in count_dict and count_dict[age_bin_interval] != 0: + outfile.write( + f"GOOD: There are {count_dict[age_bin_interval]} transmission in age bin " + f"{age_bin_interval} since transmission rate is {rate}.\n") + else: + self.success = False + outfile.write( + f"BAD: There is no transmission in age bin {age_bin_interval} but " + f"transmission rate is {rate} != 0.\n") + return self.success + + def plot_bar(self, count_dict): + # Extract keys and values from the dictionary + keys = list(count_dict.keys()) + values = list(count_dict.values()) + # Create a histogram + plt.bar(keys, values) + # Set labels for the axes + plt.xlabel('Age bin') + plt.ylabel('New Infections') + # Set the title of the graph + plt.title('Histogram of New Infection Values') + plt.savefig('new_infection_count_by_age_bin.png') + plt.close() + + +def application(output_folder="output", my_arg=None): + if not my_arg: + my_sft = AgeBaseHINTValueTest(stdout='stdout.txt') + else: + my_sft = AgeBaseHINTValueTest( + output=my_arg.output, stdout='stdout.txt', json_report=my_arg.json_report, event_csv=my_arg.event_csv, + config=my_arg.config, campaign=my_arg.campaign, report_name=my_arg.report_name, debug=my_arg.debug) + my_sft.run() + + +if __name__ == "__main__": + # execute only if run as a script + my_arg = arg_parser() + application(my_arg=my_arg) diff --git a/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process_baseline.py b/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process_baseline.py new file mode 100644 index 0000000..36f0d29 --- /dev/null +++ b/tests/sft_tests/hint_tests/test_age_base_hint_values/ep4_dir/dtk_post_process_baseline.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +import os.path +import json +import pandas as pd +import matplotlib.pyplot as plt + +from idm_test.dtk_test.sft_class import arg_parser, SFT + + +class AgeBaseHINTValueTest(SFT): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # overwrite the test method + def test(self): + self.success = True + + # Load Age Base HINT parameters from Demographics file + age_bin_edges_in_years = [0, 5, 20, 60, -1] + + # Replace the last age bin edge -1 with a large year of age. It will be used to generate the IntervalIndex for + # dataframe later. + age_bin_edges_in_years[-1] = 125 + + with open(self.report_name, "w") as outfile: + outfile.write("This simulation generate a baseline to be compared with other simulations.") + with open(os.path.join(self.output_folder, "ReportEventRecorder.csv"), 'r') as infile: + df = pd.read_csv(infile) + df.columns = df.columns.to_series().apply(lambda x: x.strip()) + # Filter the dataframe for NewInfectionEvent + new_infected_df = df.loc[(df['Event_Name'] == 'NewInfectionEvent')] + # Add a new 'Age_Year' column to the DataFrame by converting 'Age' to years. + new_infected_df['Age_Year'] = new_infected_df['Age'] / 365.0 + # Add a new 'Age_Bin' column to the DataFrame based on age bin edges. + bins = pd.IntervalIndex.from_breaks(age_bin_edges_in_years) + new_infected_df['Age_Bin'] = pd.cut(new_infected_df['Age_Year'], bins=bins, right=False).astype(str) + + # Save dataframes to csv for debugging + new_infected_df.to_csv("ReportEventRecorder_AgeBin.csv") + + # ignore the first timestep where we have the outbreak + new_infected_df = new_infected_df.loc[(new_infected_df['Year'] != 2005)] + + # Count NewInfectionEvent by Age_Bin + count_dict = {age_bin: count for age_bin, count in new_infected_df.groupby('Age_Bin').size().items()} + + # Plot the dictionary into histogram plot for debugging + if len(count_dict): + self.plot_bar(count_dict) + outfile.write(f'{count_dict}\n') + + return self.success + + def plot_bar(self, count_dict): + # Extract keys and values from the dictionary + keys = list(count_dict.keys()) + values = list(count_dict.values()) + # Create a histogram + plt.bar(keys, values) + # Set labels for the axes + plt.xlabel('Age bin') + plt.ylabel('New Infections') + # Set the title of the graph + plt.title('Histogram of New Infection Values') + plt.savefig('new_infection_count_by_age_bin.png') + plt.close() + + +def application(output_folder="output", my_arg=None): + if not my_arg: + my_sft = AgeBaseHINTValueTest(stdout='stdout.txt') + else: + my_sft = AgeBaseHINTValueTest( + output=my_arg.output, stdout='stdout.txt', json_report=my_arg.json_report, event_csv=my_arg.event_csv, + config=my_arg.config, campaign=my_arg.campaign, report_name=my_arg.report_name, debug=my_arg.debug) + my_sft.run() + + +if __name__ == "__main__": + # execute only if run as a script + my_arg = arg_parser() + application(my_arg=my_arg) diff --git a/tests/sft_tests/hint_tests/test_age_base_hint_values/test_age_base_hint_values.py b/tests/sft_tests/hint_tests/test_age_base_hint_values/test_age_base_hint_values.py new file mode 100644 index 0000000..11d4dfe --- /dev/null +++ b/tests/sft_tests/hint_tests/test_age_base_hint_values/test_age_base_hint_values.py @@ -0,0 +1,67 @@ +import os +import sys +from functools import partial + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append('../') +from hint_test_helper import TestAgeBaseHint, build_camp, build_demog, set_param_fn, Age_Bin_Edges_In_Years, \ + num_age_bin, build_demog_target_all_age_bin + + +def build_demog_age_bins(): + """ + Build a demographics input file for the DTK using emod_api with AgeDependentTransmission matrix. The transmission + matrix will have a constant value for one group and 0 for the other groups + """ + demog = build_demog() + TransmissionMatrix = [[0] * num_age_bin for _ in range(num_age_bin)] + + group_indexes = [0, 1, 2, 3] + values = [2, 1, 0.5, 0.1] + + for row in TransmissionMatrix: + for group_index in group_indexes: + row[group_index] = values[group_index] + + demog.AddAgeDependentTransmission( + Age_Bin_Edges_In_Years=Age_Bin_Edges_In_Years.copy(), + TransmissionMatrix=TransmissionMatrix + ) + + return demog + + +class TestAgeBaseHintValues(TestAgeBaseHint): + @classmethod + def setUpClass(cls): + cls.test_name = os.path.basename(__file__) + + def test_age_base_hint_values(self): + self.age_base_hint_test(build_demog_age_bins) + + + # Please run this test separately with a different version of post process script which is \test_age_base_hint_values\ep4_dir\dtk_post_process_baseline.py + def _test_age_base_hint_values_baseline(self): + # This is a baseline simulation that is run without the AgeBadeHINT feature. + self.age_base_hint_test(build_demog) + + def test_age_base_hint_values_all_0s(self): + build_demog_age_bins_all_0s = partial(build_demog_target_all_age_bin, value=0) + self.age_base_hint_test(build_demog_age_bins_all_0s) + + def test_age_base_hint_values_all_1s(self): + build_demog_age_bins_all_1s = partial(build_demog_target_all_age_bin, value=1) + self.age_base_hint_test(build_demog_age_bins_all_1s) + + def test_age_base_hint_values_all_gt1(self): + build_demog_age_bins_all_gt1 = partial(build_demog_target_all_age_bin, value=1.75) + self.age_base_hint_test(build_demog_age_bins_all_gt1) + + def test_age_base_hint_values_all_halves(self): + build_demog_age_bins_all_halves = partial(build_demog_target_all_age_bin, value=0.5) + self.age_base_hint_test(build_demog_age_bins_all_halves) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/tests/sft_tests/vaccine_tests/test_vax_rollout_coverage/ep4_dir/dtk_post_process.py b/tests/sft_tests/vaccine_tests/test_vax_rollout_coverage/ep4_dir/dtk_post_process.py index b37e027..45b2d9c 100644 --- a/tests/sft_tests/vaccine_tests/test_vax_rollout_coverage/ep4_dir/dtk_post_process.py +++ b/tests/sft_tests/vaccine_tests/test_vax_rollout_coverage/ep4_dir/dtk_post_process.py @@ -108,14 +108,26 @@ def test(self): total_birth_count = births_id.shape[0] actual_vax_coverage = total_vax_count / total_birth_count expected_coverage = self.campaign_obj['Demographic_Coverage'][2] - if not math.isclose(actual_vax_coverage, expected_coverage, abs_tol=0.05): # we allow 5% difference + # set the base tolerance to 5% for coverage == 1, + # there is an uniform delay period with 14 days time window that we need to consider. + tolerance = 0.05 + if expected_coverage != 1: + from idm_test.stats_test import cal_tolerance_binomial + # add the tolerance calculated based on binomial probabality on top of the base tolerance(discounted based on coverage) + tolerance = max(0, tolerance * (1 - (1 - expected_coverage) * 2)) + tolerance += cal_tolerance_binomial(expected_coverage * total_vax_count, expected_coverage, prob=0.05) + + if not math.isclose(actual_vax_coverage, expected_coverage, abs_tol=tolerance): self.success = False outfile.write( - f" BAD: at time step {self.campaign_obj['Start_Day'][2]}, vax coverage {actual_vax_coverage}, expected {expected_coverage}.\n") + f" BAD: at time step {self.campaign_obj['Start_Day'][2]}, vax coverage {actual_vax_coverage}," + f" expected {expected_coverage} with tolerance = {tolerance}.\n") outfile.write("Result is False.\n") else: self.success = True - outfile.write("GOOD: coverage is correct!\n") + outfile.write(f" GOOD: at time step {self.campaign_obj['Start_Day'][2]}, vax coverage " + f"{actual_vax_coverage}, expected {expected_coverage} with tolerance = {tolerance}.\n" + " GOOD: coverage is correct!\n") outfile.write("Result is True.\n") return self.success diff --git a/tests/workflow_tests/test_multiroute_HINT.py b/tests/workflow_tests/test_multiroute_HINT.py index 6881ed7..01a845e 100644 --- a/tests/workflow_tests/test_multiroute_HINT.py +++ b/tests/workflow_tests/test_multiroute_HINT.py @@ -47,7 +47,7 @@ def set_param_fn(config): print("Setting params.") config.parameters.Simulation_Type = "TYPHOID_SIM" config.parameters.Simulation_Duration = SIMULATION_DURATION_IN_YEARS * 365.0 - config.parameters.Base_Individual_Sample_Rate = 0.2 + config.parameters.Base_Individual_Sample_Rate = 1 config.parameters.Base_Year = BASE_YEAR config.parameters.Inset_Chart_Reporting_Start_Year = 2010 @@ -211,15 +211,15 @@ def build_demog(): task.handle_experiment_completion(experiment) task.get_file_from_comps(experiment.uid, ["PropertyReportTyphoid.json", "ReportTyphoidByAgeAndGender.csv"]) # Get downloaded local ReportEventRecorder.csv file path for all simulations - reporteventrecorder_downloaded = list( + reportbyageandgender_downloaded = list( glob(os.path.join(experiment.id, "**/ReportTyphoidByAgeAndGender.csv"), recursive=True)) propertyreport_downloaded = list( glob(os.path.join(experiment.id, "**/PropertyReportTyphoid.json"), recursive=True)) - reporteventrecorder_downloaded.sort() + reportbyageandgender_downloaded.sort() propertyreport_downloaded.sort() - for i in range(len(reporteventrecorder_downloaded)): + for i in range(len(reportbyageandgender_downloaded)): # read ReportEventRecorder.csv from each sim - df = pd.read_csv(reporteventrecorder_downloaded[i]) + df = pd.read_csv(reportbyageandgender_downloaded[i]) df.columns = df.columns.to_series().apply(lambda x: x.strip()) with open(propertyreport_downloaded[i], "r") as content: