Merge branch 'develop' into in-docker

.gitignore

@@ -7,3 +7,5 @@ _version.py
 .DS_Store
 test/data/*
 *.pyc
+.vscode/*
+notebooks/*.log

rsm_input_aggregator.py

@@ -5,8 +5,8 @@
 # inputs:
 #   rsm_main_dir: RSM main directory
 #   org_model_dir: Donor model directory
-#   agg_zones: Aggregated zones
-#   ext_zones: External zones
+#   num_rsm_zones: Number of RSM zones
+#   num_ext_zones: Number of external zones
 #
 # outputs:
 #   aggregated csv files
@@ -27,8 +27,8 @@
 
 rsm_main_dir = sys.argv[1]
 org_model_dir = sys.argv[2]
-agg_zones = sys.argv[3]
-ext_zones = sys.argv[4]
+num_rsm_zones = sys.argv[3]
+num_ext_zones = sys.argv[4]
 
 logging_start(
     filename=os.path.join(rsm_main_dir, "logFiles", "rsm-logging.log"), level=logging.INFO
@@ -42,8 +42,8 @@
     rsm_dir = rsm_main_dir,
     taz_cwk_file = "taz_crosswalk.csv",
     mgra_cwk_file = "mgra_crosswalk.csv",
-    agg_zones = agg_zones,
-    ext_zones = ext_zones,
+    num_rsm_zones = num_rsm_zones,
+    num_ext_zones = num_ext_zones,
     input_files = ["microMgraEquivMinutes.csv", "microMgraTapEquivMinutes.csv", 
     "walkMgraTapEquivMinutes.csv", "walkMgraEquivMinutes.csv", "bikeTazLogsum.csv",
     "bikeMgraLogsum.csv", "zone.term", "zones.park", "tap.ptype", "accessam.csv",

rsm_zone_aggregator.py

@@ -5,8 +5,8 @@
 # inputs:
 #   rsm_main_dir: RSM main directory
 #   full_model_output_dir: Donor model directory
-#   agg_zones: Aggregated zone
-#   ext_zones: External zones
+#   num_rsm_zones: Number of RSM zones
+#   num_ext_zones: Number of external zones
 # outputs:
 #   mgra_crosswalk.csv
 #   taz_crosswalk.csv
@@ -35,8 +35,8 @@
 
 rsm_main_dir = sys.argv[1]
 full_model_output_dir = sys.argv[2]
-agg_zones = int(sys.argv[3])
-ext_zones = int(sys.argv[4])
+NUM_RSM_ZONES = int(sys.argv[3])
+NUM_EXT_ZONES = int(sys.argv[4])
 
 #input files
 rsm_input_dir = os.path.join(rsm_main_dir, "input")
@@ -48,13 +48,11 @@
 FULL_ABM_SYNTH_PERSONS = os.path.join(full_model_output_dir, "input", "persons.csv")
 ABM_PROPERTIES_FOLDER = os.path.join(rsm_main_dir, "conf")
 ABM_PROPERTIES = os.path.join(ABM_PROPERTIES_FOLDER, "sandag_abm.properties")
-AGGREGATED_ZONES =  agg_zones
-EXTERNAL_ZONES = ext_zones
 EXPLICIT_ZONE_AGG = []
 
 #output files
 OUTPUT_MGRA_CROSSWALK = os.path.join(rsm_input_dir, "mgra_crosswalk.csv")
-OUTPUT_TAZ_CROSSWALK = os.path.join(rsm_input_dir, "taz_crosswalk.csv")
+OUTPUT_TAZ_CROSSWALK = os.path.join(rsm_input_dir, "taz_crosswalk.csv")`
 OUTPUT_CLUSTER_CENTROIDS = os.path.join(rsm_input_dir, "cluster_centroids.csv")
 OUTPUT_RSM_ZONE_FILE = os.path.join(rsm_input_dir, "mgra13_based_input2016.csv")
 
@@ -111,7 +109,7 @@
 agglom3full = aggregate_zones(
     tazs,
     cluster_factors=cluster_factors,
-    n_zones=AGGREGATED_ZONES,
+    n_zones=NUM_RSM_ZONES,
     method="agglom_adj",
     use_xy=1e-4,
     explicit_agg=EXPLICIT_ZONE_AGG,
@@ -147,12 +145,12 @@
     agglom3full.loc[agglom3full['hch_dist'].isin(ech_dist_mod), 'enrollgrade9to12'] = 99999
 
 
-ext_zones_df = pd.DataFrame({'taz':range(1,EXTERNAL_ZONES+1), 'cluster_id': range(1,EXTERNAL_ZONES+1)})
+ext_zones_df = pd.DataFrame({'taz':range(1,NUM_EXT_ZONES+1), 'cluster_id': range(1,NUM_EXT_ZONES+1)})
 
 taz_crosswalk = pd.concat([taz_crosswalk, ext_zones_df])
 taz_crosswalk = taz_crosswalk.sort_values('taz')
 
-mgra_crosswalk['cluster_id'] = mgra_crosswalk['cluster_id'] - EXTERNAL_ZONES
+mgra_crosswalk['cluster_id'] = mgra_crosswalk['cluster_id'] - NUM_EXT_ZONES
 
 mgra_crosswalk.to_csv(OUTPUT_MGRA_CROSSWALK, index=False)
 taz_crosswalk.to_csv(OUTPUT_TAZ_CROSSWALK, index=False)

sandag_abm/src/main/emme/toolbox/master_run.py

@@ -299,9 +299,6 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
         visualizer_reference_label = props["visualizer.reference.label"]
         visualizer_build_label = props["visualizer.build.label"]
         mgraInputFile = props["mgra.socec.file"]
-        taz_cwk_file = props["taz.to.cluster.crosswalk.file"]
-        mgra_cwk_file = props["mgra.to.cluster.crosswalk.file"]
-        cluster_zone_file = props["cluster.zone.centroid.file"]
 
         period_ids = list(enumerate(periods, start=int(scenario_id) + 1))
 
@@ -345,10 +342,13 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
 
         #RSM Inputs
         run_rsm_setup = int(props["run.rsm.setup"])
-        run_rsm_abm = int(props["run.rsm.abm.setup"])
+        run_rsm = int(props["run.rsm"])
         org_full_model_dir = props["full.modelrun.dir"]
-        aggregated_zones = props["agg.zones"]
-        ext_zones = props["external.zones"]
+        num_rsm_zones = props["rsm.zones"]
+        num_external_zones = props["external.zones"]
+        taz_crosswalk_file = props["taz.to.cluster.crosswalk.file"]
+        mgra_crosswalk_file = props["mgra.to.cluster.crosswalk.file"]
+        cluster_zone_file = props["cluster.zone.centroid.file"]
 
         #check if visualizer.reference.path is valid in filesbyyears.csv
         if not os.path.exists(visualizer_reference_path):
@@ -401,23 +401,20 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
             self.check_for_fatal(_join(self._path, "logFiles", "AtTransitCheck_event.log"),
                                  "AT and Transit network consistency checking failed! Open AtTransitCheck_event.log for details.")
 
-            #working
             if run_rsm_setup>0:
                 self.run_proc("runRSMZoneAggregator.cmd", 
-                [main_directory, rsm_venv_path, rsm_script_path, org_full_model_dir, aggregated_zones, ext_zones],
+                [main_directory, rsm_venv_path, rsm_script_path, org_full_model_dir, num_rsm_zones, num_external_zones],
                 "Zone Aggregator")
 
                 self.run_proc("runRSMInputAggregator.cmd", 
-                [main_directory, rsm_venv_path, rsm_script_path, org_full_model_dir, aggregated_zones, ext_zones], 
+                [main_directory, rsm_venv_path, rsm_script_path, org_full_model_dir, num_rsm_zones, num_external_zones], 
                 "Input Files Aggregator")
 
                 self.run_proc("runInputTripMatrixAggregator.cmd", 
-                [main_directory, rsm_python2_path, org_full_model_dir, rsm_script_path, taz_cwk_file], 
+                [main_directory, rsm_python2_path, org_full_model_dir, rsm_script_path, taz_crosswalk_file], 
                 "Input Trip Matrix files Aggregator")
-
-
-
-
+
+
             if startFromIteration == 1:  # only run the setup / init steps if starting from iteration 1
                 if not skipWalkLogsums:
                     self.run_proc("runSandagWalkLogsums.cmd", [drive, path_forward_slash],
@@ -455,8 +452,8 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
                     #########################################  added on 0629
 
 
-                    taz_cwk = pd.read_csv(os.path.join(main_directory, taz_cwk_file), index_col = 0)
-                    taz_cwk = taz_cwk['cluster_id'].to_dict()
+                    taz_crosswalk = pd.read_csv(os.path.join(main_directory, taz_crosswalk_file), index_col = 0)
+                    taz_crosswalk = taz_crosswalk['cluster_id'].to_dict()
 
                     emmebank = _m.Modeller().emmebank
                     scenario = emmebank.scenario(base_scenario)
@@ -488,7 +485,7 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
                             length.append(link.length)
 
                     df = pd.DataFrame({'links' : links, 'i_nodes' : i_nodes, 'j_nodes': j_nodes, 'ul1_org': data1, 'length_org':length})
-                    df['i_nodes_new'] = df['i_nodes'].map(taz_cwk)
+                    df['i_nodes_new'] = df['i_nodes'].map(taz_crosswalk)
 
                     #get XY of existing centroids
                     j_nodes_list = df['j_nodes'].unique()
@@ -741,15 +738,16 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
                 if not skipCoreABM[iteration]:
                     self.remove_prev_iter_files(core_abm_files, output_dir, iteration)
 
-                    if run_rsm_abm>0:
+                    if run_rsm:
 
-                        #set accessibility to false
+                        #set 'acc.read.input.file' property to false
+                        #For RSM, accessibility is now run outside the main ABM run. 
                         self.run_proc(
                         "setAccessibility.cmd",
                         [main_directory, rsm_venv_path, rsm_script_path, "false"],
                         "Modify sandag_abm.properties file for accessibility", capture_output=True)
 
-                        #creating accessibility file
+                        #run accessibility
                         self.run_proc(
                         "runSandagAbm_acc.cmd",
                         [drive, drive + path_forward_slash, sample_rate[iteration], msa_iteration],
@@ -760,13 +758,14 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
                         [main_directory, rsm_venv_path, rsm_script_path, msa_iteration], 
                         "Create sampled households and person files")
 
-                        #set accessibility to true
+                        #set 'acc.read.input.file' property to true
+                        #so that the accessibilities (computed before) are read from file during the CT-RAMP ABM run
                         self.run_proc(
                         "setAccessibility.cmd",
                         [main_directory, rsm_venv_path, rsm_script_path, "true"],
                         "Modify sandag_abm.properties file for accessibility", capture_output=True)
 
-                        #run CT RAMP
+                        #run CT-RAMP
                         self.run_proc(
                         "runSandagAbm_RSM_SDRM.cmd",
                         [drive, drive + path_forward_slash, sample_rate[iteration], msa_iteration],

sandag_abm/src/main/resources/runRSMInputAggregator.cmd

@@ -4,8 +4,8 @@ set MAIN_DIR=%1
 set PYTHON_DIR=%2
 set RSM_SCRIPT_DIR=%3
 set ORG_FULL_MODEL_DIR=%4
-set AGGREGATED_ZONES=%5
-set EXT_ZONES=%6
+set NUM_RSM_ZONES=%5
+set NUM_EXT_ZONES=%6
 
 rem #### create aggregated input/uec files
-"%PYTHON_DIR%/python.exe" "%RSM_SCRIPT_DIR%/rsm_input_aggregator.py" %MAIN_DIR% %ORG_FULL_MODEL_DIR% %AGGREGATED_ZONES% %EXT_ZONES%
+"%PYTHON_DIR%/python.exe" "%RSM_SCRIPT_DIR%/rsm_input_aggregator.py" %MAIN_DIR% %ORG_FULL_MODEL_DIR% %NUM_RSM_ZONES% %NUM_EXT_ZONES%

sandag_abm/src/main/resources/runRSMZoneAggregator.cmd

@@ -4,10 +4,10 @@ set MAIN_DIR=%1
 set PYTHON_DIR=%2
 set RSM_SCRIPT_DIR=%3
 set ORG_FULL_MODEL_DIR=%4
-set AGGREGATED_ZONES=%5
-set EXT_ZONES=%6
+set NUM_RSM_ZONES=%5
+set NUM_EXT_ZONES=%6
 
 rem #### create RSM zones and TAZ/MGRA to RSM zone crosswalks
-"%PYTHON_DIR%/python.exe" "%RSM_SCRIPT_DIR%/rsm_zone_aggregator.py" %MAIN_DIR% %ORG_FULL_MODEL_DIR% %AGGREGATED_ZONES% %EXT_ZONES%
+"%PYTHON_DIR%/python.exe" "%RSM_SCRIPT_DIR%/rsm_zone_aggregator.py" %MAIN_DIR% %ORG_FULL_MODEL_DIR% %NUM_RSM_ZONES% %NUM_EXT_ZONES%