Merge branch 'develop' into seea-recursive

# Conflicts:
#	flowsa/__init__.py
#	flowsa/flowbyactivity.py
#	flowsa/methods/flowbysectormethods/CAP_HAP_national_2017.yaml
#	flowsa/methods/flowbysectormethods/README.md

.github/workflows/generate_FBS.yml

@@ -0,0 +1,50 @@
+# This workflow will generate all FlowBySector files and store as artifact
+
+name: Generate FBS
+
+on:
+  pull_request:
+    branches: [master]
+    types: [opened, reopened, ready_for_review]   # excludes syncronize to avoid redundant trigger from commits on PRs
+  workflow_dispatch:            # also allow manual trigger, for testing purposes
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+
+    - name: Update pip & install testing pkgs
+      run: |
+        python -VV
+        python -m pip install --upgrade pip setuptools wheel
+        pip install pytest
+
+    # install package & dependencies
+    - name: Install package and dependencies
+      run: |
+        pip install .
+
+    - name: Generate FBS
+      run: |
+        pytest -m generate_fbs
+
+    - name: Upload files
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        # Artifact name
+        name: FlowBySector
+        # A file, directory or wildcard patter that describes what to upload
+        path: | # uses local user data dir for ubuntu
+          ~/.local/share/flowsa/FlowBySector/*
+          ~/.local/share/stewi/Log/*
+        if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
+        # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.

.github/workflows/python-app.yml

@@ -61,4 +61,4 @@ jobs:
 
     - name: Test with pytest
       run: |
-        pytest --doctest-modules
+        pytest --doctest-modules -m "not generate_fbs"

.github/workflows/test_methods.yml

@@ -34,7 +34,7 @@ jobs:
 
     - name: Test FBA config
       run: |
-        python flowsa/test_FBA_urls.py
+        python flowsa/test_methods.py
 
     - name: Compare FBS with remote
       id: FBS
@@ -50,6 +50,9 @@ jobs:
         # Artifact name
         name: FBS diff files
         # A file, directory or wildcard patter that describes what to upload
-        path: ${{ env.LD_LIBRARY_PATH }}/python3.10/site-packages/flowsa/data/fbs_diff/*_diff.csv
+        path: |
+            ${{ env.LD_LIBRARY_PATH }}/python3.10/site-packages/flowsa/data/fbs_diff/*_diff.csv
+            ~/.local/share/flowsa/FlowBySector/*
+            ~/.local/share/flowsa/Log/*
         if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
         # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.

LICENSE

@@ -1,116 +1,21 @@
-CC0 1.0 Universal
-
-Statement of Purpose
-
-The laws of most jurisdictions throughout the world automatically confer
-exclusive Copyright and Related Rights (defined below) upon the creator and
-subsequent owner(s) (each and all, an "owner") of an original work of
-authorship and/or a database (each, a "Work").
-
-Certain owners wish to permanently relinquish those rights to a Work for the
-purpose of contributing to a commons of creative, cultural and scientific
-works ("Commons") that the public can reliably and without fear of later
-claims of infringement build upon, modify, incorporate in other works, reuse
-and redistribute as freely as possible in any form whatsoever and for any
-purposes, including without limitation commercial purposes. These owners may
-contribute to the Commons to promote the ideal of a free culture and the
-further production of creative, cultural and scientific works, or to gain
-reputation or greater distribution for their Work in part through the use and
-efforts of others.
-
-For these and/or other purposes and motivations, and without any expectation
-of additional consideration or compensation, the person associating CC0 with a
-Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
-and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
-and publicly distribute the Work under its terms, with knowledge of his or her
-Copyright and Related Rights in the Work and the meaning and intended legal
-effect of CC0 on those rights.
-
-1. Copyright and Related Rights. A Work made available under CC0 may be
-protected by copyright and related or neighboring rights ("Copyright and
-Related Rights"). Copyright and Related Rights include, but are not limited
-to, the following:
-
-  i. the right to reproduce, adapt, distribute, perform, display, communicate,
-  and translate a Work;
-
-  ii. moral rights retained by the original author(s) and/or performer(s);
-
-  iii. publicity and privacy rights pertaining to a person's image or likeness
-  depicted in a Work;
-
-  iv. rights protecting against unfair competition in regards to a Work,
-  subject to the limitations in paragraph 4(a), below;
-
-  v. rights protecting the extraction, dissemination, use and reuse of data in
-  a Work;
-
-  vi. database rights (such as those arising under Directive 96/9/EC of the
-  European Parliament and of the Council of 11 March 1996 on the legal
-  protection of databases, and under any national implementation thereof,
-  including any amended or successor version of such directive); and
-
-  vii. other similar, equivalent or corresponding rights throughout the world
-  based on applicable law or treaty, and any national implementations thereof.
-
-2. Waiver. To the greatest extent permitted by, but not in contravention of,
-applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
-unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
-and Related Rights and associated claims and causes of action, whether now
-known or unknown (including existing as well as future claims and causes of
-action), in the Work (i) in all territories worldwide, (ii) for the maximum
-duration provided by applicable law or treaty (including future time
-extensions), (iii) in any current or future medium and for any number of
-copies, and (iv) for any purpose whatsoever, including without limitation
-commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
-the Waiver for the benefit of each member of the public at large and to the
-detriment of Affirmer's heirs and successors, fully intending that such Waiver
-shall not be subject to revocation, rescission, cancellation, termination, or
-any other legal or equitable action to disrupt the quiet enjoyment of the Work
-by the public as contemplated by Affirmer's express Statement of Purpose.
-
-3. Public License Fallback. Should any part of the Waiver for any reason be
-judged legally invalid or ineffective under applicable law, then the Waiver
-shall be preserved to the maximum extent permitted taking into account
-Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
-is so judged Affirmer hereby grants to each affected person a royalty-free,
-non transferable, non sublicensable, non exclusive, irrevocable and
-unconditional license to exercise Affirmer's Copyright and Related Rights in
-the Work (i) in all territories worldwide, (ii) for the maximum duration
-provided by applicable law or treaty (including future time extensions), (iii)
-in any current or future medium and for any number of copies, and (iv) for any
-purpose whatsoever, including without limitation commercial, advertising or
-promotional purposes (the "License"). The License shall be deemed effective as
-of the date CC0 was applied by Affirmer to the Work. Should any part of the
-License for any reason be judged legally invalid or ineffective under
-applicable law, such partial invalidity or ineffectiveness shall not
-invalidate the remainder of the License, and in such case Affirmer hereby
-affirms that he or she will not (i) exercise any of his or her remaining
-Copyright and Related Rights in the Work or (ii) assert any associated claims
-and causes of action with respect to the Work, in either case contrary to
-Affirmer's express Statement of Purpose.
-
-4. Limitations and Disclaimers.
-
-  a. No trademark or patent rights held by Affirmer are waived, abandoned,
-  surrendered, licensed or otherwise affected by this document.
-
-  b. Affirmer offers the Work as-is and makes no representations or warranties
-  of any kind concerning the Work, express, implied, statutory or otherwise,
-  including without limitation warranties of title, merchantability, fitness
-  for a particular purpose, non infringement, or the absence of latent or
-  other defects, accuracy, or the present or absence of errors, whether or not
-  discoverable, all to the greatest extent permissible under applicable law.
-
-  c. Affirmer disclaims responsibility for clearing rights of other persons
-  that may apply to the Work or any use thereof, including without limitation
-  any person's Copyright and Related Rights in the Work. Further, Affirmer
-  disclaims responsibility for obtaining any necessary consents, permissions
-  or other rights required for any use of the Work.
-
-  d. Affirmer understands and acknowledges that Creative Commons is not a
-  party to this document and has no duty or obligation with respect to this
-  CC0 or use of the Work.
-
-For more information, please see
-<http://creativecommons.org/publicdomain/zero/1.0/>
+MIT License
+
+Copyright (c) 2022 U.S. Environmental Protection Agency
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,38 +1,58 @@
 # flowsa
-`flowsa` is a data processing library that attributes resource use, waste, emissions, and loss to economic sectors. `flowsa` aggregates, combines, and allocates data from a variety of sources. The sources can be found in the [GitHub wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-activity-datasets) under "Flow-By-Activity Datasets".
-
-`flowsa` helps support [USEEIO](https://www.epa.gov/land-research/us-environmentally-extended-input-output-useeio-technical-content) as part of the [USEEIO modeling](https://www.epa.gov/land-research/us-environmentally-extended-input-output-useeio-models) framework. The USEEIO models estimate potential impacts of goods and services in the US economy. The [Flow-By-Sector datasets](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-sector-datasets) created in FLOWSA are the environmental inputs to [`useeior`](https://github.com/USEPA/useeior).
+`flowsa` is a data processing library attributing resources (environmental, 
+monetary, and human), wastes, emissions, and losses to sectors, typically 
+[NAICS codes](https://www.census.gov/naics/). `flowsa` aggregates, combines,
+and allocates data from a variety of sources. The sources can be found in the 
+[GitHub wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-activity-datasets) 
+under "Flow-By-Activity Datasets".
+
+`flowsa` helps support 
+[USEEIO](https://www.epa.gov/land-research/us-environmentally-extended-input-output-useeio-technical-content) 
+as part of the [USEEIO modeling](https://www.epa.gov/land-research/us-environmentally-extended-input-output-useeio-models) 
+framework. The USEEIO models estimate potential impacts of goods and 
+services in the US economy. The 
+[Flow-By-Sector datasets](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-sector-datasets) 
+created in FLOWSA are the environmental inputs to 
+[`useeior`](https://github.com/USEPA/useeior).
 
 ## Usage
-### Flow-By-Activity Datasets
-Flow-By-Activity datasets are formatted tables from a variety of sources. They are largely unchanged from the original data source, with the exception of formatting.
+### Flow-By-Activity (FBA) Datasets
+Flow-By-Activity datasets are formatted tables from a variety of sources. 
+They are largely unchanged from the original data source, with the 
+exception of formatting. A list of available FBA datasets can be found in 
+the [Wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-activity-datasets).
 
 `import flowsa` \
 `flowsa.seeAvailableFlowByModels('FBA')` \
 `flowsa.getFlowByActivity(datasource="USDA_CoA_Cropland", year=2017)`
 
-### Flow-By-Sector Datasets
-Flow-By-Sector datasets are tables of environmental and other data attributed to [sectors](https://www.census.gov/naics/).
+### Flow-By-Sector (FBS) Datasets
+Flow-By-Sector datasets are tables of environmental and other data 
+attributed to [sectors](https://www.census.gov/naics/). A list of available 
+FBS datasets can be found in the [Wiki](https://github.com/USEPA/flowsa/wiki/Available-Data#flow-by-sector-datasets).
 
 `import flowsa` \
 `flowsa.seeAvailableFlowByModels('FBS')` \
 `flowsa.getFlowBySector('Water_national_2015_m1')`
 
 ## Installation
-`pip install git+https://github.com/USEPA/flowsa.git@v1.1#egg=flowsa`
+`pip install git+https://github.com/USEPA/flowsa.git@vX.X.X#egg=flowsa`
 
-where v1.1 can be replaced with the version you wish to install under 
+where vX.X.X can be replaced with the version you wish to install under 
 [Releases](https://github.com/USEPA/flowsa/releases).
 
 ### Additional Information on Installation, Examples, Detailed Documentation
 For more information on `flowsa` see the [wiki](https://github.com/USEPA/flowsa/wiki).
 
 ## Disclaimer
 
-The United States Environmental Protection Agency (EPA) GitHub project code is provided on an "as is" basis
-and the user assumes responsibility for its use.  EPA has relinquished control of the information and no longer
-has responsibility to protect the integrity , confidentiality, or availability of the information.  Any
-reference to specific commercial products, processes, or services by service mark, trademark, manufacturer,
-or otherwise, does not constitute or imply their endorsement, recommendation or favoring by EPA.  The EPA seal
-and logo shall not be used in any manner to imply endorsement of any commercial product or activity by EPA or
+The United States Environmental Protection Agency (EPA) GitHub project code 
+is provided on an "as is" basis and the user assumes responsibility for its 
+use. EPA has relinquished control of the information and no longer has 
+responsibility to protect the integrity, confidentiality, or availability 
+of the information. Any reference to specific commercial products, 
+processes, or services by service mark, trademark, manufacturer, or 
+otherwise, does not constitute or imply their endorsement, recommendation 
+or favoring by EPA. The EPA seal and logo shall not be used in any manner 
+to imply endorsement of any commercial product or activity by EPA or
 the United States Government.

flowsa/README.md

@@ -2,18 +2,24 @@
 Python scripts used to generate Flow-By-Activity (FBA)
 and Flow-By-Sector (FBS) datasets
 
-1. _"_init_.py"_
+1. _"\_\_init\_\_.py"_
 2. _"allocation.py"_
 3. _"bibliography.py"_
 4. _"common.py"_
 5. _"dataclean.py"_
-6. _"fbs_allocation.py"_
-7. _"flowbyactivity.py"_
-8. _"flowbyfunctions.py"_
-9. _"flowbysector.py"_
-10. _"literature_values.py"_
-11. _"metadata.py"_
-12. _"sectormapping.py"_
-13. _"settings.py"_
-14. _"test_examples.py"_
-15. _"validation.py"_
+6. _"datavisualization.py"_
+7. _"fbs_allocation.py"_
+8. _"flowbyactivity.py"_
+9. _"flowbyfunctions.py"_
+10. _"flowbysector.py"_
+11. _"flowsa_yaml.py"_
+12. _"literature_values.py"_
+13. _"location.py"_
+14. _"metadata.py"_
+15. _"schema.py"_
+16. _"sectormapping.py"_
+17. _"settings.py"_
+18. _"test_examples.py"_
+19. _"test_FBS_against_remote.py"_
+20. _"test_methods.py"_
+21. _"validation.py"_

flowsa/__init__.py

@@ -41,6 +41,10 @@ def getFlowByActivity(datasource, year, flowclass=None, geographic_level=None,
         year=int(year),
         download_ok=download_FBA_if_missing
     )
+
+    if len(fba) ==0:
+        raise flowsa.exceptions.FBANotAvailableError(
+            message=f"Error generating {datasource} for {str(year)}")
     if flowclass is not None:
         fba = fba.query('Class == @flowclass')
     # if geographic level specified, only load rows in geo level