Merge pull request #367 from USEPA/flowsa2.0-release

Flowsa2.0 release

.github/workflows/compare_single_FBS.yml

@@ -8,7 +8,7 @@ on:
         method:
             description: "FBS"
             required: true
-            default: "CAP_HAP_national_2017"
+            default: "CAP_HAP_national_2017_m1"
 
 jobs:
   build:
@@ -39,14 +39,14 @@ jobs:
           METHOD: ${{ github.event.inputs.method }}
       run: |
         echo "Method: ${{ github.event.inputs.method }}"
-        python flowsa/test_single_FBS.py --method $METHOD
+        python tests/test_single_FBS.py --method $METHOD
 
     - name: Upload files
       if: always()
       uses: actions/[email protected]
       with:
         # Artifact name
-        name: FBS diff files
+        name: "${{ github.event.inputs.method }}"
         # A file, directory or wildcard patter that describes what to upload
         path: | # uses local user data dir for macos
           ~/Library/Application Support/flowsa/FBSComparisons/*

.github/workflows/generate_FBA.yml

@@ -45,7 +45,7 @@ jobs:
       run: |
         echo "Source: ${{ github.event.inputs.source }}"
         echo "Year: ${{ github.event.inputs.year }}"
-        python flowsa/test_single_FBA.py --source $SOURCE --year $YEAR
+        python tests/test_single_FBA.py --source $SOURCE --year $YEAR
 
     - name: Upload files
       if: always()

.github/workflows/python-app.yml

@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        py-version: ['3.8', '3.9', '3.10']
+        py-version: ['3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3
@@ -62,3 +62,28 @@ jobs:
     - name: Test with pytest
       run: |
         pytest --doctest-modules -m "not generate_fbs"
+
+  FBS_method_testing:
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+
+    - name: Update pip & install testing pkgs
+      run: |
+        python -VV
+        python -m pip install --upgrade pip setuptools wheel
+        pip install pytest pytest-cov flake8
+
+    # install package & dependencies
+    - name: Install package and dependencies
+      run: |
+        pip install .
+
+    - name: Test FBS config
+      run: |
+        python tests/test_FBS_methods.py

.github/workflows/test_methods.yml

@@ -34,7 +34,7 @@ jobs:
 
     - name: Test FBA config
       run: |
-        python flowsa/test_methods.py
+        python tests/test_methods.py
 
   FBS_testing:
     runs-on: macos-latest
@@ -61,9 +61,9 @@ jobs:
       id: FBS
       if: always() # Proceed even if Test FBA fails
       run: |
-        for m in $(python flowsa/test_FBS_against_remote.py list)
+        for m in $(python tests/test_FBS_against_remote.py list)
         do
-          python flowsa/test_FBS_against_remote.py "$m"
+          python tests/test_FBS_against_remote.py "$m"
         done
 
     - name: Upload csv files

.gitignore

@@ -2,8 +2,10 @@
 .idea
 venv
 .venv
+external_data.yaml
 *.env
 *__pycache__
 *.egg*
 *.cache
-**/data/fbs_diff/*
+**/data/fbs_diff/*
+flowby-old.py

README.md

@@ -4,8 +4,8 @@
 <!-- badges: end -->
 
 # flowsa
-`flowsa` is a data processing library attributing resources (environmental, 
-monetary, and human), wastes, emissions, and losses to sectors, typically 
+`flowsa` is a data processing library attributing the flows of resources 
+(environmental, monetary, and human), wastes, emissions, and losses to sectors, typically 
 [NAICS codes](https://www.census.gov/naics/). `flowsa` aggregates, combines,
 and allocates data from a variety of sources. The sources can be found in the 
 [GitHub wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-activity-datasets) 
@@ -23,22 +23,35 @@ created in FLOWSA are the environmental inputs to
 ## Usage
 ### Flow-By-Activity (FBA) Datasets
 Flow-By-Activity datasets are formatted tables from a variety of sources. 
-They are largely unchanged from the original data source, with the 
-exception of formatting. A list of available FBA datasets can be found in 
+They are largely unchanged from the original data source, except for 
+formatting. A list of available FBA datasets can be found in 
 the [Wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-activity-datasets).
 
 `import flowsa` \
+Return list of all available FBA datasets, including years 
 `flowsa.seeAvailableFlowByModels('FBA')` \
-`flowsa.getFlowByActivity(datasource="USDA_CoA_Cropland", year=2017)`
+Generate and return pandas dataframe for 2014 Energy Information 
+Administration (EIA) Manufacturing Energy Consumption Survey (MECS) land use \
+`fba = flowsa.getFlowByActivity(datasource="EIA_MECS_Land", year=2014)`
 
 ### Flow-By-Sector (FBS) Datasets
 Flow-By-Sector datasets are tables of environmental and other data 
 attributed to [sectors](https://www.census.gov/naics/). A list of available 
-FBS datasets can be found in the [Wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-sector-datasets).
+FBS datasets can be found in the 
+[Wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-sector-datasets).
 
 `import flowsa` \
+Return list of all available FBS datasets
 `flowsa.seeAvailableFlowByModels('FBS')` \
-`flowsa.getFlowBySector('Water_national_2015_m1')`
+Generate and return pandas dataframe for national water withdrawals 
+attributed to 6-digit sectors. Download all required FBA datasets from 
+Data Commons. \
+`fbs = flowsa.getFlowBySector('Water_national_2015_m1', 
+download_FBAs_if_missing=True)`
+
+### Examples
+Additional example code can be found in the [examples](https://github.
+com/USEPA/flowsa/tree/master/examples) folder.
 
 ## Installation
 `pip install git+https://github.com/USEPA/[email protected]#egg=flowsa`
@@ -49,6 +62,10 @@ where vX.X.X can be replaced with the version you wish to install under
 ### Additional Information on Installation, Examples, Detailed Documentation
 For more information on `flowsa` see the [wiki](https://github.com/USEPA/flowsa/wiki).
 
+### Accessing datsets output by FLOWSA
+FBA and FBS datasets can be accessed on 
+[EPA's Data Commons](https://dmap-data-commons-ord.s3.amazonaws.com/index.html?prefix=flowsa/) without running the Python code. 
+
 ## Disclaimer
 
 The United States Environmental Protection Agency (EPA) GitHub project code 

docs/DataQualityPedigreeMatrix.md

@@ -1,4 +1,4 @@
-# Data Quality Pedigree Mattix
+# Data Quality Pedigree Matrix
 
 The following table is an update of
 'Table 3 Updated Data Quality Pedigree Matrix – Flow Indicators' from [USEPA 2016 'Guidance on Data Quality Assessment for Life Cycle Inventory Data'](https://cfpub.epa.gov/si/si_public_record_report.cfm?Lab=NRMRL&dirEntryId=321834)

examples/generate_data_visualization.py

@@ -3,16 +3,17 @@
 # coding=utf-8
 
 """
+Data visualization will be revised at a later date to work for flowsav2.0
+
 Generate plots to explore Flow-By-Sector model outputs
 """
 
 import flowsa
-import flowsa.datavisualization as dv
 from flowsa.settings import plotoutputpath
 import matplotlib.pyplot as plt
 
 
-########## Produce facet graph of resources associated with cropland sectors ##########
+### Produce facet graph of resources associated with livestock sectors ###
 sectors = ['112']
 sector_length_display = 6
 plottype = 'facet_graph'
@@ -21,77 +22,89 @@
                'Employment 2017': 'Employment_national_2017'}
 
 
-dv.FBSscatterplot(method_dict, plottype,
+flowsa.FBSscatterplot(method_dict, plottype,
                   sector_length_display=sector_length_display,
                   sectors_to_include=sectors,
                   plot_title='Direct Resource Use for Livestock'
                   )
 # Can manually adjust the figure pop up before saving
-plt.savefig(f"{plotoutputpath}livestock_resource_use.png", dpi=300)
+plt.savefig(plotoutputpath / "livestock_resource_use.png", dpi=300)
 
 
-########## Compare the results between water method 1 and method 2 ##########
+### Compare the results between national employment for 2015 and 2018 ###
 sectors = ['21']
 sector_length_display = 6
 plottype = 'method_comparison'
-method_dict = {'Water Withdrawal 2015 M1': 'Water_national_2015_m1',
-               'Water Withdrawal 2015 M2': 'Water_national_2015_m2'}
+method_dict = {'National Employment 2015': 'Employment_national_2015',
+               'National Employment 2018': 'Employment_national_2018'}
 
-flowsa.generateFBSplot(method_dict, plottype,
-                       sector_length_display=sector_length_display,
-                       sectors_to_include=sectors,
-                       plot_title='Comparison of 2015 National Water '
-                                  'Withdrawals Method 1 and Method 2 for '
-                                  'Mining Sectors'
-                       )
+flowsa.FBSscatterplot(method_dict, plottype,
+                  sector_length_display=sector_length_display,
+                  sectors_to_include=sectors,
+                  plot_title=('Comparison of 2015 and 2018 Employment '
+                              'for Mining Sectors')
+                  )
 # Can manually adjust the figure pop up before saving
-plt.savefig(f"{plotoutputpath}mining_water_comp.png", dpi=300)
-
-
-########## Compare food waste flows via Sankey ##########
-methodnames = ['Food_Waste_national_2018_m1', 'Food_Waste_national_2018_m2']
-target_sector_level = 'NAICS_2'
-target_subset_sector_level = {
-    'Food_Waste_national_2018_m1': {'NAICS_4': ['2212'],
-                                    'NAICS_6': ['62421', '31111', '32411',
-                                                '56221', '62421', '115112',
-                                                '22132'],
-                                    'NAICS_7': ['562212', '562219']
-                                    },
-    'Food_Waste_national_2018_m2': {
-        'SectorConsumedBy': {'NAICS_6': ['115112', '22132','311119',
-                                         '32411', '562213','62421'],
-                             'NAICS_7': ['562212', '562219']
-                               }}
-}
-# set domain to scale sankey diagrams
-domain_dict = {0: {'x': [0.01, 0.49], 'y': [0, 1]},
-               1: {'x': [.51, 0.99], 'y': [.12, .88]}
-               }
-
-dv.generateSankeyDiagram(
-    methodnames,
-    target_sector_level=target_sector_level,
-    target_subset_sector_level=target_subset_sector_level,
-    use_sectordefinition=True,
-    sectors_to_include=None,
-    fbsconfigpath=None,
-    orientation='horizontal',
-    domain_dict=domain_dict,
-    value_label_format='brackets',
-    subplot_titles=['m1', 'm2'],
-    filename='FoodWasteSankey'
-)
+plt.savefig(plotoutputpath / "mining_employment_comp.png", dpi=300)
 
 
-####### GHG Bar Chart ############
+### GHG Bar Chart ###
 # Option 1 - GHG emissions by GHG
-dv.stackedBarChart('GHG_national_2018_m1',
-                   selection_fields={'Sector': ['111110', '112120', '325312']},
-                   filename='GHGEmissions')
+flowsa.stackedBarChart('GHG_national_2018_m1',
+                       generalize_AttributionSources=True,
+                       selection_fields={'SectorProducedBy': ['111', '324'],
+                                         'Flowable': ['Carbon dioxide',
+                                                      'Methane']},
+                       industry_spec={'default': 'NAICS_3'},
+                       filename='GHG_national_2018_m1_emissions_barchart',
+                       axis_title='Emissions (MMT CO2e)'
+                       )
 
 # Option 2 - specify indicator, much have LCIAformatter installed
 # https://github.com/USEPA/LCIAformatter
-dv.stackedBarChart('GHG_national_2018_m1', impact_cat='Global warming',
-                   selection_fields={'Sector': ['111110', '112120', '325312']},
-                   filename='GHGEmissionsGlobalWarming')
+flowsa.stackedBarChart('GHG_national_2018_m1',
+                       generalize_AttributionSources=True,
+                       impact_cat="Global warming",
+                       selection_fields={'SectorProducedBy': ['111', '324'],
+                                         'Flowable': ['Carbon dioxide']},
+                       industry_spec={'default': 'NAICS_3'},
+                       filename='GHG_national_2018_m1_CO2_barchart',
+                       axis_title='Global Warming Potential (MMT CO2e)'
+                       )
+
+
+# todo: will update the sankey code for recursive method post v2.0 release
+########## Compare food waste flows via Sankey ##########
+# methodnames = ['Food_Waste_national_2018_m1', 'Food_Waste_national_2018_m2']
+# target_sector_level = 'NAICS_2'
+# target_subset_sector_level = {
+#     'Food_Waste_national_2018_m1': {'NAICS_4': ['2212'],
+#                                     'NAICS_6': ['62421', '31111', '32411',
+#                                                 '56221', '62421', '115112',
+#                                                 '22132'],
+#                                     'NAICS_7': ['562212', '562219']
+#                                     },
+#     'Food_Waste_national_2018_m2': {
+#         'SectorConsumedBy': {'NAICS_6': ['115112', '22132','311119',
+#                                          '32411', '562213','62421'],
+#                              'NAICS_7': ['562212', '562219']
+#                                }}
+# }
+# # set domain to scale sankey diagrams
+# domain_dict = {0: {'x': [0.01, 0.49], 'y': [0, 1]},
+#                1: {'x': [.51, 0.99], 'y': [.12, .88]}
+#                }
+#
+# flowsa.generateSankeyDiagram(
+#     methodnames,
+#     target_sector_level=target_sector_level,
+#     target_subset_sector_level=target_subset_sector_level,
+#     use_sectordefinition=True,
+#     sectors_to_include=None,
+#     fbsconfigpath=None,
+#     orientation='horizontal',
+#     domain_dict=domain_dict,
+#     value_label_format='brackets',
+#     subplot_titles=['m1', 'm2'],
+#     filename='FoodWasteSankey'
+# )