From 446cea4845b53086f982cc779e6297859339be60 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Wed, 23 Oct 2024 18:18:58 +0000 Subject: [PATCH 1/7] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index b6afe3a5..dde386b9 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -26,8 +26,8 @@ Note: This does not include variables defined in python scripts. | channels | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | ClusteringMethod | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | cmd | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | CondaEnv | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | +| condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | condaFile | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | config_path | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | @@ -66,15 +66,15 @@ Note: This does not include variables defined in python scripts. | end_elem | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | end_element | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | end_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| EndDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_posterior', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | | endDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | +| EndDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_posterior', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | | EndDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | ErrorBCs | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ErrorOH | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | -| file | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/utilities/common.sh', 'src/inversion_scripts/run_inversion.sh'] | ['imi_failed', 'send_error'] | ['run_hemco_sa', 'print_stats', 'imi_failed', 'ncmax', 'ncmin', 'calculate_geoschem_domain', 'send_error'] | | FILE | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | +| file | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/utilities/common.sh', 'src/inversion_scripts/run_inversion.sh'] | ['imi_failed', 'send_error'] | ['run_hemco_sa', 'print_stats', 'imi_failed', 'ncmax', 'ncmin', 'calculate_geoschem_domain', 'send_error'] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | filename | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | @@ -189,8 +189,8 @@ Note: This does not include variables defined in python scripts. | nElements | ['src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/inversion_scripts/run_inversion.sh'] | ['reduce_dimension', 'setup_kf', 'run_posterior', 'calculate_num_jacobian_runs', 'setup_imi'] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_inversion', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'send_error'] | | NEW | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['activate_observations', 'setup_template'] | ['setup_imi', 'activate_observations', 'setup_template'] | | nPeriods | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | -| nRuns | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian', 'calculate_num_jacobian_runs'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | nruns | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | +| nRuns | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian', 'calculate_num_jacobian_runs'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | nTracers | ['src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh'] | ['calculate_tracer_end', 'calculate_num_jacobian_runs'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'send_error'] | | NudgeFactor | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['run_period'] | | NumberOfElements | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | @@ -276,8 +276,8 @@ Note: This does not include variables defined in python scripts. | ROOT | ['src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh'] | [] | [] | ['create_simulation_dir', 'setup_template'] | | run_dirs_pth | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | RUNDIR | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | -| RunDir | ['src/utilities/cleanup_script.sh'] | ['src/utilities/cleanup_script.sh'] | [] | [] | | runDir | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['setup_posterior', 'create_simulation_dir', 'setup_imi', 'setup_spinup', 'run_preview'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_preview'] | +| RunDir | ['src/utilities/cleanup_script.sh'] | ['src/utilities/cleanup_script.sh'] | [] | [] | | rundir_num | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | RunDirs | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_inversion', 'run_inversion', 'run_notebooks', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_hemco_prior_emis', 'run_hemco_sa', 'run_preview'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | @@ -325,8 +325,8 @@ Note: This does not include variables defined in python scripts. | start_elem | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_element | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| startDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | StartDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | +| startDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | StartDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | @@ -348,7 +348,7 @@ Note: This does not include variables defined in python scripts. | xhat | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['generate_optimized_BC_values', 'generate_optimized_OH_value'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | | xstr | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | xUSE | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | -| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | +| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | YYYY | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | yyyymmdd | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | \ No newline at end of file From f32efc74bc274520faa9e3971ede7c3c514f651d Mon Sep 17 00:00:00 2001 From: megan-he Date: Tue, 29 Oct 2024 19:27:04 -0400 Subject: [PATCH 2/7] fix jacobian scale factor calculation --- src/inversion_scripts/get_jacobian_scalefactors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inversion_scripts/get_jacobian_scalefactors.py b/src/inversion_scripts/get_jacobian_scalefactors.py index 562032ab..c639255b 100644 --- a/src/inversion_scripts/get_jacobian_scalefactors.py +++ b/src/inversion_scripts/get_jacobian_scalefactors.py @@ -49,7 +49,7 @@ def get_jacobian_scalefactors(period_number, inv_directory, ref_directory): # Note 2: This also assumes that the temporal variabily of the swapped emissions is the same. # If the temporal variability is different, there will be error associated with scaling # the Jacobian. - sf_K = pert_sf / ref_pert_sf + sf_K = ref_pert_sf / pert_sf # Apply the target_emis_ratio to the scale factors sf_K = sf_K * target_emis_ratio From f28f83514a90a4713ca40e9aa3035d7291b91466 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Tue, 29 Oct 2024 23:28:36 +0000 Subject: [PATCH 3/7] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index dde386b9..eb51ed76 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -28,11 +28,11 @@ Note: This does not include variables defined in python scripts. | cmd | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | | CondaEnv | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | -| CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | condaFile | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | +| CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | config_path | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | -| ConfigFile | ['run_imi.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/utilities/cleanup_script.sh'] | ['run_imi.sh', 'src/utilities/cleanup_script.sh'] | [] | ['setup_inversion', 'run_preview'] | | configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | +| ConfigFile | ['run_imi.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/utilities/cleanup_script.sh'] | ['run_imi.sh', 'src/utilities/cleanup_script.sh'] | [] | ['setup_inversion', 'run_preview'] | | ConfigPath | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'run_posterior', 'run_jacobian', 'run_preview'] | | constYr | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | copied_config | ['src/components/inversion_component/inversion.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_notebooks'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | @@ -174,12 +174,12 @@ Note: This does not include variables defined in python scripts. | Lons | ['src/components/template_component/template.sh'] | ['src/components/template_component/template.sh'] | ['setup_template'] | ['setup_template'] | | MakePeriodsCSV | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/setup_component/setup.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'setup_imi'] | | MaxSimultaneousRuns | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/jacobian_component/jacobian.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_jacobian'] | -| MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | Met | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['create_statevector', 'setup_imi', 'setup_template', 'run_hemco_prior_emis'] | +| MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | metDir | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | metNum | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | MM | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | +| mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | Month | ['src/utilities/crop_met.sh'] | [] | [] | [] | | n_elems | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | name | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | @@ -260,8 +260,8 @@ Note: This does not include variables defined in python scripts. | RequestedCPUs | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | | RequestedMemory | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | | RequestedTime | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | -| RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | Res | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/inversion_scripts/run_inversion.sh'] | [] | ['run_posterior', 'setup_inversion', 'setup_imi', 'setup_template', 'run_hemco_prior_emis', 'send_error'] | +| RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | resnum | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['run_hemco_prior_emis'] | ['run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | | RESOLUTION | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | response | ['src/utilities/cleanup_script.sh'] | [] | [] | [] | @@ -275,9 +275,9 @@ Note: This does not include variables defined in python scripts. | rm_command | ['src/utilities/cleanup_script.sh'] | ['src/utilities/cleanup_script.sh'] | [] | [] | | ROOT | ['src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh'] | [] | [] | ['create_simulation_dir', 'setup_template'] | | run_dirs_pth | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | -| RUNDIR | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | -| runDir | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['setup_posterior', 'create_simulation_dir', 'setup_imi', 'setup_spinup', 'run_preview'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_preview'] | | RunDir | ['src/utilities/cleanup_script.sh'] | ['src/utilities/cleanup_script.sh'] | [] | [] | +| runDir | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['setup_posterior', 'create_simulation_dir', 'setup_imi', 'setup_spinup', 'run_preview'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_preview'] | +| RUNDIR | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | | rundir_num | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | RunDirs | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_inversion', 'run_inversion', 'run_notebooks', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_hemco_prior_emis', 'run_hemco_sa', 'run_preview'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | @@ -325,8 +325,8 @@ Note: This does not include variables defined in python scripts. | start_elem | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_element | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| StartDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | | startDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | +| StartDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | | StartDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | @@ -348,7 +348,7 @@ Note: This does not include variables defined in python scripts. | xhat | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['generate_optimized_BC_values', 'generate_optimized_OH_value'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | | xstr | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | xUSE | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | -| YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | +| YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | YYYY | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | yyyymmdd | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | \ No newline at end of file From 0162b48f5b75d534ae8f9fda11d1a38ddd388c8a Mon Sep 17 00:00:00 2001 From: megan-he Date: Tue, 29 Oct 2024 19:47:30 -0400 Subject: [PATCH 4/7] Revert "fix jacobian scale factor calculation" This reverts commit f32efc74bc274520faa9e3971ede7c3c514f651d. --- src/inversion_scripts/get_jacobian_scalefactors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inversion_scripts/get_jacobian_scalefactors.py b/src/inversion_scripts/get_jacobian_scalefactors.py index c639255b..562032ab 100644 --- a/src/inversion_scripts/get_jacobian_scalefactors.py +++ b/src/inversion_scripts/get_jacobian_scalefactors.py @@ -49,7 +49,7 @@ def get_jacobian_scalefactors(period_number, inv_directory, ref_directory): # Note 2: This also assumes that the temporal variabily of the swapped emissions is the same. # If the temporal variability is different, there will be error associated with scaling # the Jacobian. - sf_K = ref_pert_sf / pert_sf + sf_K = pert_sf / ref_pert_sf # Apply the target_emis_ratio to the scale factors sf_K = sf_K * target_emis_ratio From 9fa220994a6bba09870a95d434aa274bbead5eb0 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Tue, 29 Oct 2024 23:48:32 +0000 Subject: [PATCH 5/7] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index eb51ed76..28ff81f9 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -26,10 +26,10 @@ Note: This does not include variables defined in python scripts. | channels | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | ClusteringMethod | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | cmd | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| CondaEnv | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | -| condaFile | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | +| CondaEnv | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | +| condaFile | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | config_path | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | | configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ConfigFile | ['run_imi.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/utilities/cleanup_script.sh'] | ['run_imi.sh', 'src/utilities/cleanup_script.sh'] | [] | ['setup_inversion', 'run_preview'] | @@ -73,8 +73,8 @@ Note: This does not include variables defined in python scripts. | ErrorBCs | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ErrorOH | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | -| FILE | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | | file | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/utilities/common.sh', 'src/inversion_scripts/run_inversion.sh'] | ['imi_failed', 'send_error'] | ['run_hemco_sa', 'print_stats', 'imi_failed', 'ncmax', 'ncmin', 'calculate_geoschem_domain', 'send_error'] | +| FILE | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | filename | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | @@ -93,8 +93,8 @@ Note: This does not include variables defined in python scripts. | GCVizDir | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | GEOSCHEM_VERSION | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['run_imi.sh'] | [] | ['setup_imi'] | | geosChemDataPath | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | -| geosChemEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | GEOSChemEnv | ['run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | +| geosChemEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | GlobalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | GOSAT | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/setup_component/setup.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['activate_observations'] | | gridded_posterior_filename | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['setup_posterior'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | @@ -178,8 +178,8 @@ Note: This does not include variables defined in python scripts. | MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | metDir | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | metNum | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| MM | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | +| MM | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | Month | ['src/utilities/crop_met.sh'] | [] | [] | [] | | n_elems | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | name | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | @@ -276,8 +276,8 @@ Note: This does not include variables defined in python scripts. | ROOT | ['src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh'] | [] | [] | ['create_simulation_dir', 'setup_template'] | | run_dirs_pth | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | RunDir | ['src/utilities/cleanup_script.sh'] | ['src/utilities/cleanup_script.sh'] | [] | [] | -| runDir | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['setup_posterior', 'create_simulation_dir', 'setup_imi', 'setup_spinup', 'run_preview'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_preview'] | | RUNDIR | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | +| runDir | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/write_BCs/run_boundary_conditions.sh'] | ['setup_posterior', 'create_simulation_dir', 'setup_imi', 'setup_spinup', 'run_preview'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_preview'] | | rundir_num | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | RunDirs | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_inversion', 'run_inversion', 'run_notebooks', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'activate_observations', 'setup_template', 'setup_spinup', 'run_spinup', 'run_hemco_prior_emis', 'run_hemco_sa', 'run_preview'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | @@ -325,8 +325,8 @@ Note: This does not include variables defined in python scripts. | start_elem | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_element | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| startDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | StartDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'create_simulation_dir', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | +| startDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | StartDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | @@ -348,7 +348,7 @@ Note: This does not include variables defined in python scripts. | xhat | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['generate_optimized_BC_values', 'generate_optimized_OH_value'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | | xstr | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | xUSE | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | -| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | +| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | YYYY | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | yyyymmdd | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | \ No newline at end of file From f4ba1ee2d6d4cc826c0f60fa64ce224c282dff1d Mon Sep 17 00:00:00 2001 From: Megan He <59985465+megan-he@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:02:14 -0400 Subject: [PATCH 6/7] fix jacobian scale factor calculation (#285) --- src/inversion_scripts/get_jacobian_scalefactors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inversion_scripts/get_jacobian_scalefactors.py b/src/inversion_scripts/get_jacobian_scalefactors.py index 562032ab..c639255b 100644 --- a/src/inversion_scripts/get_jacobian_scalefactors.py +++ b/src/inversion_scripts/get_jacobian_scalefactors.py @@ -49,7 +49,7 @@ def get_jacobian_scalefactors(period_number, inv_directory, ref_directory): # Note 2: This also assumes that the temporal variabily of the swapped emissions is the same. # If the temporal variability is different, there will be error associated with scaling # the Jacobian. - sf_K = pert_sf / ref_pert_sf + sf_K = ref_pert_sf / pert_sf # Apply the target_emis_ratio to the scale factors sf_K = sf_K * target_emis_ratio From 1e44c58a4cd0ab9ac88b22f3ff5fc6a747e603e3 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Wed, 30 Oct 2024 01:02:36 +0000 Subject: [PATCH 7/7] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index 28ff81f9..535355e8 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -26,13 +26,13 @@ Note: This does not include variables defined in python scripts. | channels | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | ClusteringMethod | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | cmd | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | CondaEnv | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | -| CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | +| condaEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | | condaFile | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | [] | [] | +| CondaFile | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | config_path | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | -| configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ConfigFile | ['run_imi.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/utilities/cleanup_script.sh'] | ['run_imi.sh', 'src/utilities/cleanup_script.sh'] | [] | ['setup_inversion', 'run_preview'] | +| configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ConfigPath | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'run_posterior', 'run_jacobian', 'run_preview'] | | constYr | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | copied_config | ['src/components/inversion_component/inversion.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_notebooks'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | @@ -66,15 +66,15 @@ Note: This does not include variables defined in python scripts. | end_elem | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | end_element | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | end_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| endDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | EndDate | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['setup_kf', 'run_posterior', 'run_jacobian', 'setup_imi', 'setup_template', 'setup_spinup', 'run_hemco_prior_emis'] | +| endDate | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | | EndDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | ErrorBCs | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | ErrorOH | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | -| file | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/utilities/common.sh', 'src/inversion_scripts/run_inversion.sh'] | ['imi_failed', 'send_error'] | ['run_hemco_sa', 'print_stats', 'imi_failed', 'ncmax', 'ncmin', 'calculate_geoschem_domain', 'send_error'] | | FILE | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_bkgd_simulation.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | +| file | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/utilities/common.sh', 'src/inversion_scripts/run_inversion.sh'] | ['imi_failed', 'send_error'] | ['run_hemco_sa', 'print_stats', 'imi_failed', 'ncmax', 'ncmin', 'calculate_geoschem_domain', 'send_error'] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | filename | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | @@ -93,8 +93,8 @@ Note: This does not include variables defined in python scripts. | GCVizDir | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | ['send_error'] | | GEOSCHEM_VERSION | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['run_imi.sh'] | [] | ['setup_imi'] | | geosChemDataPath | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | -| GEOSChemEnv | ['run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | geosChemEnv | ['src/write_BCs/config_boundary_conditions.yml', 'src/write_BCs/run_boundary_conditions.sh'] | ['src/write_BCs/config_boundary_conditions.yml'] | [] | [] | +| GEOSChemEnv | ['run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | | GlobalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | GOSAT | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/setup_component/setup.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['activate_observations'] | | gridded_posterior_filename | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['setup_posterior'] | ['setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value'] | @@ -178,8 +178,8 @@ Note: This does not include variables defined in python scripts. | MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | metDir | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | metNum | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['setup_template', 'run_hemco_prior_emis'] | ['setup_template', 'run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | -| mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | MM | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | +| mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | Month | ['src/utilities/crop_met.sh'] | [] | [] | [] | | n_elems | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['calculate_tracer_end'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | | name | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['envs/Harvard-Cannon/imi_env.yml', 'src/components/jacobian_component/jacobian.sh'] | ['create_simulation_dir'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | @@ -189,8 +189,8 @@ Note: This does not include variables defined in python scripts. | nElements | ['src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/setup_component/setup.sh', 'src/inversion_scripts/run_inversion.sh'] | ['reduce_dimension', 'setup_kf', 'run_posterior', 'calculate_num_jacobian_runs', 'setup_imi'] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix', 'setup_posterior', 'run_posterior', 'generate_optimized_BC_values', 'generate_optimized_OH_value', 'setup_inversion', 'setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'setup_imi', 'activate_observations', 'send_error'] | | NEW | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['activate_observations', 'setup_template'] | ['setup_imi', 'activate_observations', 'setup_template'] | | nPeriods | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | -| nruns | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | nRuns | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian', 'calculate_num_jacobian_runs'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number'] | +| nruns | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['get_last_rundir_suffix'] | ['setup_kf', 'run_kf', 'run_period', 'get_last_rundir_suffix'] | | nTracers | ['src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh'] | ['calculate_tracer_end', 'calculate_num_jacobian_runs'] | ['setup_jacobian', 'create_simulation_dir', 'add_new_tracer', 'run_jacobian', 'generate_BC_perturb_values', 'calculate_tracer_end', 'calculate_num_jacobian_runs', 'is_number', 'send_error'] | | NudgeFactor | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/kalman_component/kalman.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['run_period'] | | NumberOfElements | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | [] | @@ -260,8 +260,8 @@ Note: This does not include variables defined in python scripts. | RequestedCPUs | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | | RequestedMemory | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | | RequestedTime | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/statevector_component/statevector.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/spinup_component/spinup.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/components/preview_component/preview.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml'] | [] | ['reduce_dimension', 'run_posterior', 'run_inversion', 'run_jacobian', 'run_spinup', 'run_hemco_sa', 'run_preview'] | -| Res | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/inversion_scripts/run_inversion.sh'] | [] | ['run_posterior', 'setup_inversion', 'setup_imi', 'setup_template', 'run_hemco_prior_emis', 'send_error'] | | RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | +| Res | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/hemco_prior_emis_component/hemco_prior_emis.sh', 'src/inversion_scripts/run_inversion.sh'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml', 'envs/Harvard-Cannon/config.harvard-cannon.global_inv.yml', 'src/inversion_scripts/run_inversion.sh'] | [] | ['run_posterior', 'setup_inversion', 'setup_imi', 'setup_template', 'run_hemco_prior_emis', 'send_error'] | | resnum | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['src/components/hemco_prior_emis_component/hemco_prior_emis.sh'] | ['run_hemco_prior_emis'] | ['run_hemco_prior_emis', 'run_hemco_sa', 'exclude_soil_sink'] | | RESOLUTION | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | response | ['src/utilities/cleanup_script.sh'] | [] | [] | [] |