Merge pull request #3 from camsys/main

Sharrow Run Fail: School Escorting
ActivitySim · Apr 23, 2024 · ad384fa · ad384fa
2 parents b0c839e + 92eab1d
commit ad384fa
Show file tree

Hide file tree

Showing 58 changed files with 4,612 additions and 175 deletions.
diff --git a/.github/workflow/ci.yml → .github/workflows/ci.yml b/.github/workflow/ci.yml → .github/workflows/ci.yml
@@ -50,38 +50,26 @@ jobs:
 
       - uses: actions/cache@v4
         with:
-          path: /usr/share/miniconda3/envs/asim-test
+          path: miniconda3/envs/asim-test
           key: linux-64-conda-${{ hashFiles('activitysim/conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
         id: cache
 
       - name: Update environment
         run: |
           mamba env update -n asim-test -f activitysim/conda-environments/github-actions-tests.yml
-          mamba install --yes \
-          	"psutil=5.9.5" \
-          	"pydantic=2.6.1" \
-          	"pypyr=5.8.0" \
-          	"pytables=3.6.1" \
-          	"pytest-cov" \
-          	"pytest-regressions=2.5.0" \
-          	"scikit-learn=1.2.2" \
-          	"sharrow>=2.6.0" \
-          	"simwrapper=1.8.5" \
-          	"xarray=2023.2.0" \
-          	"zarr=2.14.2" \
-          	"zstandard=0.21.0"
         if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Install activitysim
         # installing without dependencies is faster, we trust that all needed dependencies
         # are in the conda environment defined above.  Also, this avoids pip getting
         # confused and reinstalling tables (pytables).
         run: |
-          python -m pip install -e ./activitysim --no-deps
+          python -m pip install ./activitysim --no-deps
       - name: Conda checkup
         run: |
           mamba info -a
           mamba list
       - name: Test this implementation
         run: |
-          python -m pytest sandag-abm3-example/test
+          cd activitysim-sandag-abm3/test
+          python -m pytest test_sandag_abm3.py::test_sandag_abm3_progressive
diff --git a/configs/resident/atwork_subtour_destination.csv b/configs/resident/atwork_subtour_destination.csv
@@ -1,8 +1,8 @@
 Label,Description,Expression,atwork
 ,,"[email protected](skims['DIST'], 20)",1
-,,_DIST_SQUARED@_DIST**2,1
-,,_DIST_CUBED@_DIST**3,1
-,,[email protected](_DIST + 0.001),1
+ATWORK_DIST_SQUARED,,_DIST_SQUARED@_DIST**2,1
+ATWORK_DIST_CUBED,,_DIST_CUBED@_DIST**3,1
+ATWORK_DIST_LOGGED,,[email protected](_DIST + 0.001),1
 util_Distance,Distance,@_DIST,coef_dist_atwork
 util_Distance_squared,Distance_squared,@_DIST_SQUARED,coef_distsqrd_atwork
 util_Distance_fulltime_worker,Distance_fulltime_worker,@_DIST * (df.ptype==1),coef_dist_ftworker_atwork

diff --git a/configs/resident/atwork_subtour_frequency.csv b/configs/resident/atwork_subtour_frequency.csv
@@ -4,7 +4,7 @@ util_dummy_for_non_full_time_worker,pemploy!=1,coefficient_dummy_for_non_full_ti
 util_dummy_for_non_workers,"ptype in [4, 5]",coefficient_dummy_for_non_workers_no_subtours,coefficient_dummy_for_non_workers_eat,coefficient_dummy_for_non_workers_business1,coefficient_dummy_for_non_workers_maint,coefficient_dummy_for_non_workers_business2,coefficient_dummy_for_non_workers_eat_business
 util_medium_hh_income_dummy,income_segment == 2,coefficient_medium_hh_income_dummy_no_subtours,coefficient_medium_hh_income_dummy_eat,coefficient_medium_hh_income_dummy_business1,coefficient_medium_hh_income_dummy_maint,coefficient_medium_hh_income_dummy_business2,coefficient_medium_hh_income_dummy_eat_business
 util_high_hh_income_dummy,(income_segment > 2) & (income_segment < 5),coefficient_high_hh_income_dummy_no_subtours,coefficient_high_hh_income_dummy_eat,coefficient_high_hh_income_dummy_business1,coefficient_high_hh_income_dummy_maint,coefficient_high_hh_income_dummy_business2,coefficient_high_hh_income_dummy_eat_business
-util_zero_cars_owned_by_hh_dummy, auto_ownership == 0,coefficient_zero_cars_owned_by_hh_dummy_no_subtours,coefficient_zero_cars_owned_by_hh_dummy_eat,coefficient_zero_cars_owned_by_hh_dummy_business1,coefficient_zero_cars_owned_by_hh_dummy_maint,coefficient_zero_cars_owned_by_hh_dummy_business2,coefficient_zero_cars_owned_by_hh_dummy_eat_business
+util_zero_cars_owned_by_hh_dummy,auto_ownership==0,coefficient_zero_cars_owned_by_hh_dummy_no_subtours,coefficient_zero_cars_owned_by_hh_dummy_eat,coefficient_zero_cars_owned_by_hh_dummy_business1,coefficient_zero_cars_owned_by_hh_dummy_maint,coefficient_zero_cars_owned_by_hh_dummy_business2,coefficient_zero_cars_owned_by_hh_dummy_eat_business
 util_individual_discretionary_tours_made_by_full_time_worker,@(df.pemploy==1)*df.num_discr_tours,coefficient_individual_discretionary_tours_made_by_full_time_worker_no_subtours,coefficient_individual_discretionary_tours_made_by_full_time_worker_eat,coefficient_individual_discretionary_tours_made_by_full_time_worker_business1,coefficient_individual_discretionary_tours_made_by_full_time_worker_maint,coefficient_individual_discretionary_tours_made_by_full_time_worker_business2,coefficient_individual_discretionary_tours_made_by_full_time_worker_eat_business
 util_individual_discretionary_tours_made_by_part_time_worker,@(df.pemploy==2)*df.num_discr_tours,coefficient_individual_discretionary_tours_made_by_part_time_worker_no_subtours,coefficient_individual_discretionary_tours_made_by_part_time_worker_eat,coefficient_individual_discretionary_tours_made_by_part_time_worker_business1,coefficient_individual_discretionary_tours_made_by_part_time_worker_maint,coefficient_individual_discretionary_tours_made_by_part_time_worker_business2,coefficient_individual_discretionary_tours_made_by_part_time_worker_eat_business
 util_individual_eating_out_tours_made_by_person,num_eatout_tours,coefficient_individual_eating_out_tours_made_by_person_no_subtours,coefficient_individual_eating_out_tours_made_by_person_eat,coefficient_individual_eating_out_tours_made_by_person_business1,coefficient_individual_eating_out_tours_made_by_person_maint,coefficient_individual_eating_out_tours_made_by_person_business2,coefficient_individual_eating_out_tours_made_by_person_eat_business

diff --git a/configs/resident/cdap.yaml b/configs/resident/cdap.yaml
@@ -45,3 +45,6 @@ annotate_households:
   DF: households
   TABLES:
     - persons
+
+compute_settings:
+  sharrow_skip: true
diff --git a/configs/resident/cdap_joint_tour_coefficients.csv b/configs/resident/cdap_joint_tour_coefficients.csv
@@ -10,7 +10,7 @@ Label,description,Expression,dependency,coefficient
 ,person x is kid and DAP is N,ptype_px > 6,N_px,1.6898
 ,Accessibility to retail employment/Non-Mandatory Attractions,shopping_accessibility_p1,,0.055031985
 ,Income less than $30k,income_p1 < 30000,,-0.192506367
-,Income between $60k and $100k,(income_p1  >= 60000) & (income_p1 < =100000),,0.104325349
+,Income between $60k and $100k,(income_p1>=60000) & (income_p1<=100000),,0.104325349
 ,Income more than $100k,income_p1 > 100000,,0.104325349
 ,No Car Households,auto_ownership_p1 == 0,,0
 ,Cars Less than Workers,auto_ownership_p1 < num_workers_p1,,0.088402389

diff --git a/configs/resident/external_non_mandatory_identification.csv b/configs/resident/external_non_mandatory_identification.csv
@@ -1,6 +1,6 @@
 Label,Description,Expression,external_tour,internal_tour
 util_dist_to_nearest_ext_station,Distance to nearest external station,dist_to_external_zone,coef_dist_to_nearest_ext_station,
-util_log_size,Log size of nearest station,"@np.log1p(reindex(land_use.external_nonwork, df.closest_external_zone))",coef_log_size_of_nearest_ext_station,
+util_log_size,Log size of nearest station,@np.log1p(df.reindex_land_use_external_nonwork_zone_to_closest_external_zone),coef_log_size_of_nearest_ext_station,
 util_escort,escort tour ASC,"@np.where(df.tour_type == 'escort', 1, 0)",coef_escort,
 util_shopping,shopping tour ASC,"@np.where(df.tour_type == 'shopping', 1, 0)",coef_shopping,
 util_othmaint,othmaint tour ASC,"@np.where(df.tour_type == 'othmaint', 1, 0)",coef_othmaint,

diff --git a/configs/resident/external_non_mandatory_identification.yaml b/configs/resident/external_non_mandatory_identification.yaml
@@ -9,3 +9,9 @@ LOGIT_TYPE: MNL
 EXTERNAL_COL_NAME: is_external_tour
 # set to True if not external but CHOOSER_FILTER_COLUMN_NAME is True
 INTERNAL_COL_NAME: is_internal_tour
+
+preprocessor:
+  SPEC: external_non_mandatory_identification_preprocessor.csv
+  DF: df
+  TABLES:
+    - land_use
diff --git a/configs/resident/external_non_mandatory_identification_preprocessor.csv b/configs/resident/external_non_mandatory_identification_preprocessor.csv
@@ -0,0 +1,4 @@
+Description,Target,Expression
+reindex_land_use_external_nonwork_zone_to_closest_external_zone,reindex_land_use_external_nonwork_zone_to_closest_external_zone,"reindex(land_use.external_nonwork, df.closest_external_zone)"
+
+
diff --git a/configs/resident/external_student_identification.yaml b/configs/resident/external_student_identification.yaml
@@ -12,4 +12,4 @@ CHOOSER_FILTER_COLUMN_NAME: is_student
 # Adds this column to the persons table and is set to True if workplace is external
 EXTERNAL_COL_NAME: is_external_student
 # set to True if not external but CHOOSER_FILTER_COLUMN_NAME is True
-INTERNAL_COL_NAME: is_internal_student
+INTERNAL_COL_NAME: is_internal_student
diff --git a/configs/resident/external_worker_identification.csv b/configs/resident/external_worker_identification.csv
@@ -1,6 +1,6 @@
 Label,Description,Expression,work_external,work_internal
 util_dist_to_nearest_ext_station,Distance to nearest external station,dist_to_external_zone,coef_dist_to_nearest_ext_station,
-util_size_of_nearest_ext_station,Size of nearest external station,"@np.log1p(reindex(land_use.external_work,df.closest_external_zone))",coef_size_of_nearest_ext_station,
+util_size_of_nearest_ext_station,Size of nearest external station,"@np.log1p(df.reindex_land_use_external_to_work_external_zone)",coef_size_of_nearest_ext_station,
 util_dist_lt_2p5,Distance less than 2.5 miles,"@np.where(df.dist_to_external_zone<2.5,1,0)",coef_dist_lt_2p5,
 util_part_time,Part time worker,"@np.where(df.is_parttime_worker==""TRUE"",1,0)",coef_part_time,
 util_inc_lt15,Household Income less than $15k,@(df.income<15000),coef_inc_lt15,

diff --git a/configs/resident/external_worker_identification.yaml b/configs/resident/external_worker_identification.yaml
@@ -12,3 +12,9 @@ CHOOSER_FILTER_COLUMN_NAME: is_out_of_home_worker
 EXTERNAL_COL_NAME: is_external_worker
 # set to True if not external but CHOOSER_FILTER_COLUMN_NAME is True
 INTERNAL_COL_NAME: is_internal_worker
+
+preprocessor:
+  SPEC: external_worker_identification_preprocessor.csv
+  DF: df
+  TABLES:
+    - land_use
diff --git a/configs/resident/external_worker_identification_preprocessor.csv b/configs/resident/external_worker_identification_preprocessor.csv
@@ -0,0 +1,4 @@
+Description,Target,Expression
+reindex_land_use_external_work_external_zone,reindex_land_use_external_to_work_external_zone,"reindex(land_use.external_work,df.closest_external_zone)"
+
+
diff --git a/configs/resident/external_workplace_location.yaml b/configs/resident/external_workplace_location.yaml
@@ -50,4 +50,4 @@ SEGMENT_IDS:
 # MODELED_SIZE_TABLE: workplace_modeled_size
 
 # not loaded if commented out
-# SAVED_SHADOW_PRICE_TABLE_NAME: workplace_shadow_prices.csv
+# SAVED_SHADOW_PRICE_TABLE_NAME: workplace_shadow_prices.csv
diff --git a/configs/resident/joint_tour_frequency_composition.yaml b/configs/resident/joint_tour_frequency_composition.yaml
@@ -32,4 +32,7 @@ ALTS_TABLE_STRUCTURE:
     VALUE_MAP:
       1: adults
       2: children
-      3: mixed
+      3: mixed
+
+compute_settings:
+  sharrow_skip: true
diff --git a/configs/resident/joint_tour_scheduling.yaml b/configs/resident/joint_tour_scheduling.yaml
@@ -11,3 +11,6 @@ preprocessor:
     - land_use
     - households
     - joint_tour_participants
+
+compute_settings:
+  sharrow_skip: true
diff --git a/configs/resident/mandatory_tour_scheduling.yaml b/configs/resident/mandatory_tour_scheduling.yaml
@@ -30,6 +30,10 @@ TOUR_SPEC_SEGMENTS:
   school: school
   univ: univ
 
+preprocessor:
+  SPEC: mandatory_tour_scheduling_annotate_choosers_preprocessor.csv
+  DF: df
+
 ALTS_PREPROCESSOR:
   work:
     SPEC: mandatory_tour_scheduling_annotate_alts_preprocessor.csv

diff --git a/configs/resident/mandatory_tour_scheduling_annotate_choosers_preprocessor.csv b/configs/resident/mandatory_tour_scheduling_annotate_choosers_preprocessor.csv
@@ -0,0 +1,3 @@
+Description,Target,Expression
+subsequent_tour_is_work,subsequent_tour_is_work,df.tour_type.shift(-1) == 'work'
+subsequent_tour_is_school,subsequent_tour_is_school,df.tour_type.shift(-1) == 'school'
diff --git a/configs/resident/non_mandatory_tour_destination.csv b/configs/resident/non_mandatory_tour_destination.csv
@@ -1,9 +1,9 @@
 Label,Description,Expression,escort,shopping,eatout,othmaint,social,othdiscr
-,,"[email protected](skims['DIST'], 20)",1,1,1,1,1,1
-,,_DIST_SQUARED@_DIST**2,1,1,1,1,1,1
-,,_DIST_CUBED@_DIST**3,1,1,1,1,1,1
-,,[email protected](_DIST + 0.001),1,1,1,1,1,1
-,,"[email protected](df.tour_type == 'escort', 20, np.where(df.tour_type == 'othmaint', 8, 10))",1,1,1,1,1,1
+DIST,,"[email protected](skims['DIST'], 20)",1,1,1,1,1,1
+DIST_SQUARED,,_DIST_SQUARED@_DIST**2,1,1,1,1,1,1
+DIST_CUBED,,_DIST_CUBED@_DIST**3,1,1,1,1,1,1
+DIST_LOGGED,,[email protected](_DIST + 0.001),1,1,1,1,1,1
+CALIB_DIST,,"[email protected](df.tour_type == 'escort', 20, np.where(df.tour_type == 'othmaint', 8, 10))",1,1,1,1,1,1
 util_dist,dist,@_DIST,coef_dist_escort,coef_dist_shopping,coef_dist_eatout,coef_dist_maint,coef_dist_social,coef_dist_discr
 util_dist_squared,dist_squared,@_DIST_SQUARED,coef_distsqrd_escort,coef_distsqrd_shopping,coef_distsqrd_eatout,coef_distsqrd_maint,coef_distsqrd_social,coef_distsqrd_discr
 util_dist_cubed,dist_cubed,@_DIST_CUBED,coef_zero,coef_distcubed_shopping,coef_distcubed_eatout,coef_zero,coef_distcubed_social,coef_distcubed_discr

diff --git a/configs/resident/non_mandatory_tour_scheduling.yaml b/configs/resident/non_mandatory_tour_scheduling.yaml
@@ -51,23 +51,35 @@ TOUR_SPEC_SEGMENTS:
 # spec keyed by <spec_segment>
 SPEC_SEGMENTS:
   escort:
-    'SPEC': tour_scheduling_nonmandatory_escort.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_escort_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_escort.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_escort_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
   shopping:
-    'SPEC': tour_scheduling_nonmandatory_shopping.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_shopping_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_shopping.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_shopping_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
   eatout:
-    'SPEC': tour_scheduling_nonmandatory_eatout.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_eatout_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_eatout.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_eatout_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
   othdiscr:
-    'SPEC': tour_scheduling_nonmandatory_othdiscr.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_othdiscr_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_othdiscr.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_othdiscr_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
   othmaint:
-    'SPEC': tour_scheduling_nonmandatory_othmaint.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_othmaint_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_othmaint.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_othmaint_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
   social:
-    'SPEC': tour_scheduling_nonmandatory_social.csv
-    'COEFFICIENTS': tour_scheduling_nonmandatory_social_coefficients.csv
+    SPEC: tour_scheduling_nonmandatory_social.csv
+    COEFFICIENTS: tour_scheduling_nonmandatory_social_coefficients.csv
+    compute_settings:
+      sharrow_skip: true
 
 ## alts preprocessor keyed by <spec_segment>
 #ALTS_PREPROCESSOR:
@@ -91,3 +103,6 @@ SPEC_SEGMENTS:
 #    DF: alt_tdd
 #
 DESTINATION_FOR_TOUR_PURPOSE: destination
+
+compute_settings:
+  sharrow_skip: true
diff --git a/configs/resident/parking_location_choice.csv b/configs/resident/parking_location_choice.csv
@@ -1,7 +1,7 @@
 Description,Expression,no_segmentation
 Person is worker,_person_is_worker@((df.ptype == 1) | (df.ptype == 2)),1
 Trip is work purpose,_work_trip@(df.purpose == 'work'),1
-Parking to destination distance skim,_walk_distance@pd_skims['DISTWALK'],1
+Parking to destination distance skim,_walk_distance@pd_skims['walkTime'] / 60 * walkSpeed,1
 #,,
 # actual utility terms starts from here,,
 Log of parking size,"@np.where(df.parking_spaces>0,np.log(df.parking_spaces),-999)",1

diff --git a/configs/resident/school_escorting.yaml b/configs/resident/school_escorting.yaml
@@ -45,3 +45,6 @@ SIMULATE_CHOOSER_COLUMNS:
 CONSTANTS:
   max_bin_difference_between_departure_times: 1
   mins_per_time_bin: 30
+
+compute_settings:
+  sharrow_skip: true
diff --git a/configs/resident/settings.yaml b/configs/resident/settings.yaml
@@ -26,8 +26,6 @@ want_dest_choice_sample_tables: False
 # households with all tour types
 # trace_hh_id: 1051657
 
-chunk_training_mode: disabled
-
 # trace origin, destination in accessibility calculation; comment out or leave empty for no trace
 #trace_od:
 
@@ -46,6 +44,8 @@ input_table_list:
       mgra: home_zone_id
       hinc: income
       hht: HHT
+    recode_columns:
+      home_zone_id: land_use.zone_id
     keep_columns:
       - home_zone_id
       - income
@@ -58,8 +58,6 @@ input_table_list:
   - tablename: persons
     filename: persons.csv
     index_col: person_id
-    rename_columns:
-      PERID: person_id
     rename_columns:
       hhid: household_id
       perid: person_id
@@ -80,6 +78,9 @@ input_table_list:
     index_col: zone_id
     rename_columns:
       MAZ: zone_id
+    recode_columns:
+      zone_id: zero-based
+      TAZ: zero-based
     drop_columns:
       - i1
       - i2
@@ -127,8 +128,6 @@ distributed_time_factor_nonwork_stddev: 0.6
 distributed_time_factor_min: 0.1
 distributed_time_factor_max: 10
 
-resume_after:
-
 models:
   ### mp_init_proto_pop (single process)
   - initialize_proto_population # Separate step so proto tables can be split for multiprocess.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@
		Description,Target,Expression
		reindex_land_use_external_nonwork_zone_to_closest_external_zone,reindex_land_use_external_nonwork_zone_to_closest_external_zone,"reindex(land_use.external_nonwork, df.closest_external_zone)"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@
		Description,Target,Expression
		reindex_land_use_external_work_external_zone,reindex_land_use_external_to_work_external_zone,"reindex(land_use.external_work,df.closest_external_zone)"