Merge pull request #365 from NREL/348-upload-buildstockcsv-to-s3-duri…

…ng-postprocessing

upload-buildstockcsv-to-s3

buildstockbatch/base.py

@@ -871,7 +871,9 @@ def process_results(self, skip_combine=False, force_upload=False):
 
         aws_conf = self.cfg.get('postprocessing', {}).get('aws', {})
         if 's3' in aws_conf or force_upload:
-            s3_bucket, s3_prefix = postprocessing.upload_results(aws_conf, self.output_dir, self.results_dir)
+            s3_bucket, s3_prefix = postprocessing.upload_results(
+                aws_conf, self.output_dir, self.results_dir, self.sampler.csv_path
+            )
             if 'athena' in aws_conf:
                 postprocessing.create_athena_tables(aws_conf, os.path.basename(self.output_dir), s3_bucket, s3_prefix)
 

buildstockbatch/postprocessing.py

@@ -592,7 +592,7 @@ def remove_intermediate_files(fs, results_dir, keep_individual_timeseries=False)
         fs.rm(ts_in_dir, recursive=True)
 
 
-def upload_results(aws_conf, output_dir, results_dir):
+def upload_results(aws_conf, output_dir, results_dir, buildstock_csv_filename):
     logger.info("Uploading the parquet files to s3")
 
     output_folder_name = Path(output_dir).name
@@ -622,14 +622,25 @@ def upload_results(aws_conf, output_dir, results_dir):
         logger.error(f"There are already {n_existing_files} files in the s3 folder {s3_bucket}/{s3_prefix_output}.")
         raise FileExistsError(f"s3://{s3_bucket}/{s3_prefix_output}")
 
-    def upload_file(filepath):
-        full_path = parquet_dir.joinpath(filepath)
+    def upload_file(filepath, s3key=None):
+        full_path = filepath if filepath.is_absolute() else parquet_dir.joinpath(filepath)
         s3 = boto3.resource('s3')
         bucket = s3.Bucket(s3_bucket)
-        s3key = Path(s3_prefix_output).joinpath(filepath).as_posix()
+        if s3key is None:
+            s3key = Path(s3_prefix_output).joinpath(filepath).as_posix()
         bucket.upload_file(str(full_path), str(s3key))
 
-    dask.compute(map(dask.delayed(upload_file), all_files))
+    tasks = list(map(dask.delayed(upload_file), all_files))
+    if buildstock_csv_filename is not None:
+        buildstock_csv_filepath = Path(buildstock_csv_filename)
+        if buildstock_csv_filepath.exists():
+            tasks.append(dask.delayed(upload_file)(
+                buildstock_csv_filepath,
+                f"{s3_prefix_output}buildstock_csv/{buildstock_csv_filepath.name}"
+            ))
+        else:
+            logger.warning(f"{buildstock_csv_filename} doesn't exist, can't upload.")
+    dask.compute(tasks)
     logger.info(f"Upload to S3 completed. The files are uploaded to: {s3_bucket}/{s3_prefix_output}")
     return s3_bucket, s3_prefix_output
 

buildstockbatch/test/conftest.py

@@ -3,6 +3,7 @@
 import shutil
 import tempfile
 import yaml
+from pathlib import Path
 
 OUTPUT_FOLDER_NAME = 'output'
 

buildstockbatch/test/test_base.py

@@ -13,13 +13,14 @@
 import tempfile
 from unittest.mock import patch, MagicMock, PropertyMock
 import yaml
+from pathlib import Path
 
 import buildstockbatch
 from buildstockbatch.base import BuildStockBatchBase
 from buildstockbatch.local import LocalBatch
 from buildstockbatch.exc import ValidationError
 from buildstockbatch.postprocessing import write_dataframe_as_parquet
-from buildstockbatch.utils import read_csv
+from buildstockbatch.utils import read_csv, ContainerRuntime
 
 dask.config.set(scheduler='synchronous')
 here = os.path.dirname(os.path.abspath(__file__))
@@ -128,10 +129,16 @@ def test_upload_files(mocked_boto3, basic_residential_project_file):
     mocked_boto3.client = MagicMock(return_value=mocked_glueclient)
     mocked_boto3.resource().Bucket().objects.filter.side_effect = [[], ['a', 'b', 'c']]
     project_filename, results_dir = basic_residential_project_file(upload_config)
+    buildstock_csv_path = Path(results_dir).parent / 'openstudio_buildstock' / 'project_resstock_national' / 'housing_characteristics' / 'buildstock.csv'  # noqa: E501
+    shutil.copy2(
+        Path(__file__).parent / 'test_results' / 'housing_characteristics' / 'buildstock.csv',
+        buildstock_csv_path
+    )
     with patch.object(BuildStockBatchBase, 'weather_dir', None), \
             patch.object(BuildStockBatchBase, 'output_dir', results_dir), \
             patch.object(BuildStockBatchBase, 'get_dask_client') as get_dask_client_mock, \
-            patch.object(BuildStockBatchBase, 'results_dir', results_dir):
+            patch.object(BuildStockBatchBase, 'results_dir', results_dir), \
+            patch.object(BuildStockBatchBase, 'CONTAINER_RUNTIME', ContainerRuntime.LOCAL_OPENSTUDIO):
         bsb = BuildStockBatchBase(project_filename)
         bsb.process_results()
         get_dask_client_mock.assert_called_once()
@@ -200,6 +207,11 @@ def test_upload_files(mocked_boto3, basic_residential_project_file):
     assert (source_file_path, s3_file_path) in files_uploaded
     files_uploaded.remove((source_file_path, s3_file_path))
 
+    s3_file_path = s3_path + 'buildstock_csv/buildstock.csv'
+    source_file_path = str(buildstock_csv_path)
+    assert (source_file_path, s3_file_path) in files_uploaded
+    files_uploaded.remove((source_file_path, s3_file_path))
+
     assert len(files_uploaded) == 0, f"These files shouldn't have been uploaded: {files_uploaded}"
 
 

buildstockbatch/test/test_results/housing_characteristics/buildstock.csv

@@ -0,0 +1,6 @@
+Building,Usage Level,Orientation,Eaves,Door Area,Overhangs,Interior Shading,Doors,Hot Water Fixtures,Hot Water Distribution,Solar Hot Water,HVAC Secondary Heating Type And Fuel,HVAC Secondary Heating Efficiency,HVAC System Is Faulted,Dehumidifier,Refrigerator Usage Level,Cooking Range Usage Level,Dishwasher Usage Level,Clothes Washer Usage Level,Clothes Dryer Usage Level,Lighting Interior Use,Lighting Other Use,Holiday Lighting,Plug Load Diversity,Electric Vehicle,Misc Gas Fireplace,Misc Gas Grill,Misc Gas Lighting,Misc Well Pump,Natural Ventilation,Mechanical Ventilation,Bathroom Spot Vent Hour,Range Spot Vent Hour,ASHRAE IECC Climate Zone 2004,County and PUMA,CEC Climate Zone,County,AHS Region,ASHRAE IECC Climate Zone 2004 - 2A Split,State,Location Region,Census Division,Census Division RECS,Census Region,ISO RTO Region,Building America Climate Zone,PUMA,Geometry Building Type ACS,Vintage,Vintage ACS,Geometry Building Type RECS,Heating Fuel,Geometry Building Number Units SFA,Geometry Building Horizontal Location SFA,Geometry Foundation Type,Corridor,Vacancy Status,Tenure,Income,Income RECS2015,Income RECS2020,Neighbors,Radiant Barrier,Roof Material,Insulation Slab,Insulation Floor,Insulation Foundation Wall,Insulation Rim Joist,Water Heater In Unit,Water Heater Fuel,Water Heater Efficiency,HVAC Heating Type,HVAC Cooling Type,HVAC Has Shared System,HVAC Shared Efficiencies,HVAC Heating Efficiency,HVAC Has Zonal Electric Heating,HVAC Heating Type And Fuel,HVAC Cooling Efficiency,HVAC System Single Speed AC Airflow,HVAC System Single Speed ASHP Airflow,HVAC System Single Speed AC Charge,HVAC System Single Speed ASHP Charge,Heating Setpoint,Cooling Setpoint,Heating Setpoint Has Offset,Cooling Setpoint Has Offset,Heating Setpoint Offset Magnitude,Cooling Setpoint Offset Magnitude,Heating Setpoint Offset Period,Cooling Setpoint Offset Period,Ceiling Fan,Lighting,Plug Loads,HVAC Has Ducts,Ducts,Has PV,PV System Size,PV Orientation,Battery,PUMA Metro Status,Geometry Floor Area,Geometry Floor Area Bin,Geometry Stories,Geometry Building Type Height,Geometry Building Level MF,Geometry Building Number Units MF,Geometry Building Horizontal Location MF,Geometry Stories Low Rise,Geometry Attic Type,Geometry Story Bin,Geometry Wall Type,Geometry Wall Exterior Finish,Geometry Garage,Bedrooms,Occupants,Federal Poverty Level,Area Median Income,Window Areas,Insulation Ceiling,Insulation Roof,Insulation Wall,Windows,HVAC Cooling Partial Space Conditioning,Refrigerator,Cooking Range,Dishwasher,Clothes Washer Presence,Clothes Washer,Clothes Dryer,Misc Extra Refrigerator,Misc Freezer,Misc Hot Tub Spa,Misc Pool,Misc Pool Heater,Misc Pool Pump,Infiltration,REEDS Balancing Area,Generation And Emissions Assessment Region,City,AIANNH Area,Household Has Tribal Persons
+1,High,South,2 ft,20 ft^2,None,"Summer = 0.7, Winter = 0.85",Fiberglass,200% Usage,Uninsulated,None,None,None,No,None,105% Usage,120% Usage,120% Usage,120% Usage,120% Usage,100% Usage,100% Usage,No Exterior Use,200%,None,None,None,None,None,"Cooling Season, 7 days/wk",None,Hour5,Hour19,3A,"G4500510, G45001102",None,"SC, Horry County",Non-CBSA South Atlantic,3A,SC,CR09,South Atlantic,South Atlantic,South,None,Hot-Humid,"SC, 01102",Single-Family Detached,2000s,2000-09,Single-Family Detached,Electricity,None,None,Slab,Not Applicable,Occupied,Owner,80000-99999,80000-99999,60000-99999,Left/Right at 15ft,No,Composition Shingles,Uninsulated,None,None,None,Yes,Electricity,Electric Standard,Ducted Heat Pump,Ducted Heat Pump,None,None,"ASHP, SEER 13, 7.7 HSPF",No,Electricity ASHP,Ducted Heat Pump,None,None,None,None,68F,72F,No,No,0F,0F,None,None,Standard Efficiency,100% Incandescent,106%,Yes,"20% Leakage, R-8",No,None,None,None,"In metro area, not/partially in principal city",1500-1999,1500-2499,1,Single-Family Detached,None,None,None,1,Vented Attic,<8,Wood Frame,"Vinyl, Light",2 Car,3,2,400%+,150%+,F12 B12 L12 R12,R-30,"Unfinished, Uninsulated","Wood Stud, R-11","Double, Clear, Metal, Air",100% Conditioned,EF 19.9,Electric Resistance,318 Rated kWh,Yes,EnergyStar,Electric,None,None,None,None,None,None,15 ACH50,96,SRVCc,Not in a census Place,No,No
+2,Low,North,2 ft,20 ft^2,None,"Summer = 0.7, Winter = 0.85",Fiberglass,50% Usage,Uninsulated,None,None,None,No,None,95% Usage,80% Usage,80% Usage,80% Usage,80% Usage,100% Usage,100% Usage,No Exterior Use,50%,None,None,None,None,None,"Cooling Season, 7 days/wk",None,Hour22,Hour14,5A,"G1700310, G17003525",None,"IL, Cook County","CBSA Chicago-Naperville-Elgin, IL-IN-WI",5A,IL,CR04,East North Central,East North Central,Midwest,PJM,Cold,"IL, 03525",50 or more Unit,2000s,2000-09,Multi-Family with 5+ Units,Electricity,None,None,Slab,Double-Loaded Interior,Occupied,Renter,200000+,140000+,150000+,27,None,"Asphalt Shingles, Medium","2ft R10 Perimeter, Vertical",None,None,None,No,Electricity,Electric Standard,Non-Ducted Heating,Room AC,None,None,"Electric Baseboard, 100% Efficiency",Yes,Electricity Baseboard,"Room AC, EER 10.7",None,None,None,None,70F,70F,No,No,0F,0F,None,None,Standard Efficiency,100% Incandescent,89%,No,None,No,None,None,None,"In metro area, principal city",1000-1499,0-1499,4,"Multifamily with 5+ units, 4-7 stories",Middle,67,Middle,4+,None,<8,Wood Frame,"Brick, Medium/Dark",None,2,2,400%+,150%+,F30 B30 L30 R30,None,"Finished, R-38","Wood Stud, R-11","Double, Clear, Metal, Air",40% Conditioned,EF 17.6,Electric Resistance,318 Rated kWh,Yes,Standard,Electric,None,None,None,None,None,None,15 ACH50,80,RFCWc,"IL, Chicago",No,No
+3,Medium,East,2 ft,20 ft^2,None,"Summer = 0.7, Winter = 0.85",Fiberglass,100% Usage,Uninsulated,None,None,None,No,None,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,No Exterior Use,100%,None,None,None,None,None,"Cooling Season, 7 days/wk",None,Hour21,Hour17,2A,"G0100030, G01002600",None,"AL, Baldwin County",Non-CBSA East South Central,"2A - FL, GA, AL, MS",AL,CR09,East South Central,East South Central,South,None,Hot-Humid,"AL, 02600",Single-Family Detached,2000s,2000-09,Single-Family Detached,Electricity,None,None,Slab,Not Applicable,Occupied,Owner,80000-99999,80000-99999,60000-99999,Left/Right at 15ft,No,Composition Shingles,Uninsulated,None,None,None,Yes,Electricity,Electric Standard,Ducted Heat Pump,Ducted Heat Pump,None,None,"ASHP, SEER 15, 8.5 HSPF",No,Electricity ASHP,Ducted Heat Pump,None,None,None,None,70F,76F,No,No,0F,0F,None,None,Standard Efficiency,100% Incandescent,110%,Yes,"30% Leakage, R-8",No,None,None,None,"In metro area, not/partially in principal city",2000-2499,1500-2499,2,Single-Family Detached,None,None,None,2,Vented Attic,<8,Wood Frame,"Brick, Medium/Dark",2 Car,3,2,400%+,120-150%,F9 B9 L9 R9,R-30,"Unfinished, Uninsulated","Wood Stud, R-11","Double, Clear, Metal, Air",100% Conditioned,EF 19.9,Electric Resistance,318 Rated kWh,Yes,Standard,Electric,None,None,None,None,None,None,15 ACH50,90,SRSOc,In another census Place,No,No
+4,Medium,Southeast,2 ft,20 ft^2,None,"Summer = 0.7, Winter = 0.85",Fiberglass,100% Usage,Uninsulated,None,None,None,No,None,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,No Exterior Use,100%,None,None,None,None,Typical Efficiency,"Cooling Season, 7 days/wk",None,Hour23,Hour18,4A,"G3600610, G36003805",None,"NY, New York County","CBSA New York-Newark-Jersey City, NY-NJ-PA",4A,NY,CR07,Middle Atlantic,Middle Atlantic,Northeast,NYISO,Mixed-Humid,"NY, 03805",50 or more Unit,1960s,1960-79,Multi-Family with 5+ Units,Natural Gas,None,None,Vented Crawlspace,Double-Loaded Interior,Occupied,Renter,200000+,140000+,150000+,12,None,"Asphalt Shingles, Medium",None,Uninsulated,Uninsulated,Uninsulated,No,Natural Gas,Natural Gas Standard,Non-Ducted Heating,Room AC,Heating Only,"Boiler Baseboards Heating Only, Natural Gas",Shared Heating,No,Natural Gas Shared Heating,"Room AC, EER 10.7",None,None,None,None,72F,70F,No,No,0F,0F,None,None,None,100% Incandescent,89%,No,None,No,None,None,None,"In metro area, principal city",500-749,0-1499,21,"Multifamily with 5+ units, 8+ stories",Middle,326,Middle,4+,None,8+,Steel Frame,"Aluminum, Light",None,1,2,400%+,150%+,F30 B30 L30 R30,None,"Finished, R-30","Wood Stud, Uninsulated","Double, Clear, Metal, Air",40% Conditioned,EF 17.6,Gas,318 Rated kWh,None,None,None,None,None,None,None,None,None,20 ACH50,127,NYSTc,"NY, New York",No,No
+5,Medium,West,2 ft,20 ft^2,None,"Summer = 0.7, Winter = 0.85",Fiberglass,100% Usage,Uninsulated,None,None,None,No,None,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,100% Usage,No Exterior Use,100%,None,None,None,None,None,"Cooling Season, 7 days/wk",None,Hour6,Hour16,3B,"G0400150, G04000600",None,"AZ, Mohave County",Non-CBSA Mountain,3B,AZ,CR10,Mountain,Mountain South,West,None,Hot-Dry,"AZ, 00600",Single-Family Detached,2000s,2000-09,Single-Family Detached,Electricity,None,None,Slab,Not Applicable,Occupied,Owner,80000-99999,80000-99999,60000-99999,Left/Right at 15ft,No,"Tile, Clay or Ceramic",Uninsulated,None,None,None,Yes,Electricity,Electric Standard,Ducted Heating,Central AC,None,None,"Electric Furnace, 100% AFUE",No,Electricity Electric Furnace,"AC, SEER 13",None,None,None,None,68F,78F,Yes,No,3F,0F,Night +5h,None,Standard Efficiency,100% CFL,84%,Yes,"10% Leakage, R-8",No,None,None,None,Not/partially in metro area,2000-2499,1500-2499,1,Single-Family Detached,None,None,None,1,Vented Attic,<8,Wood Frame,"Wood, Medium/Dark",2 Car,4,2,400%+,150%+,F15 B15 L15 R15,R-30,"Unfinished, Uninsulated","Wood Stud, R-11","Double, Low-E, Non-metal, Air, M-Gain",100% Conditioned,EF 19.9,Electric Resistance,318 Rated kWh,Yes,Standard,Electric,None,None,None,None,None,None,5 ACH50,27,AZNMc,In another census Place,No,No

docs/changelog/changelog_dev.rst

@@ -16,7 +16,7 @@ Development Changelog
         github issues. These will be automatically linked in the documentation.
 
     .. change::
-        :tags: general, feature
+        :tags: general, bugfix
         :pullreq: 387
         :tickets: 385
 
@@ -45,3 +45,9 @@ Development Changelog
         For the Residential HPXML Workflow Generator, fixes new ``include_annual_resilience`` argument to true and
         adds a new optional ``include_timeseries_resilience`` argument that defaults to false. Also fixes new
         ``include_annual_bills`` argument to true and ``include_monthly_bills`` argument to false.
+
+    .. change::
+        :tags: postprocessing, feature
+        :pullreq: 365
+
+        Upload buildstock.csv to S3 during postprocessing