From c0d3d4125e971dc5e48b41a4ab75767ace277457 Mon Sep 17 00:00:00 2001 From: Rob Hammond Date: Tue, 13 Oct 2020 15:11:36 -0600 Subject: [PATCH] Merge develop into master - v2.0.1 (#109) * create workaround for unknow zsh issue * update documentation and GH Actions workflows * removing geopandas installation from setup.py and moving import statement in AssetData class * modifying ENGIE project pathname in exmaples so auto data extract works * minor documentation updates * Unfreezing sphinx dependencies * undoing that change * updating changelog with version 2.0.0 and unreleased * replace geopandas with pyproj (#107) * replace `GeoPandas` with `pyproj` and `Shapely` * adding `pyproj` and `shapely` to `setup.py` and `requirements.txt` * Update `python-ci.yml`to include Windows tests * update software version to 2.0.1 Co-authored-by: Rob * Update AssetData docstring Removed any lingering references to `GeoPandas`. * Remove GDAL/Windows comments * Update CHANGELOG.md * Update readme.md Co-authored-by: Rob Co-authored-by: Jordan Perr-Sauer Co-authored-by: ejsimley Co-authored-by: ejsimley <40040961+ejsimley@users.noreply.github.com> --- .github/workflows/python-ci.yml | 12 +++++++---- CHANGELOG.md | 7 ++++++- examples/00_toolkit_examples.ipynb | 4 ++-- examples/02_plant_aep_analysis.ipynb | 4 ++-- .../02b_augmented_plant_aep_analysis.ipynb | 6 +++--- examples/03_turbine_ideal_energy.ipynb | 6 +++--- examples/04_electrical_losses.ipynb | 4 ++-- examples/05_eya_gap_analysis.ipynb | 4 ++-- operational_analysis/__init__.py | 2 +- operational_analysis/types/asset.py | 20 +++++++++---------- pytest.ini | 2 ++ readme.md | 18 +++-------------- requirements.txt | 3 ++- setup.py | 5 +++-- test/regression/__init__.py | 0 .../{ => regression}/int_electrical_losses.py | 0 test/{ => regression}/int_eya_gap_analysis.py | 0 .../int_pruf_plant_analysis.py | 0 .../int_turbine_long_term_gross_energy.py | 0 test/unit/__init__.py | 0 test/{ => unit}/test_filter_toolkit.py | 0 test/{ => unit}/test_imputing_toolkit.py | 0 .../test_met_data_processing_toolkit.py | 0 test/{ => unit}/test_ml_toolkit.py | 0 test/{ => unit}/test_power_curve_toolkit.py | 0 test/{ => unit}/test_timeseries_toolkit.py | 0 .../test_unit_conversion_toolkit.py | 0 27 files changed, 48 insertions(+), 49 deletions(-) create mode 100644 pytest.ini create mode 100644 test/regression/__init__.py rename test/{ => regression}/int_electrical_losses.py (100%) rename test/{ => regression}/int_eya_gap_analysis.py (100%) rename test/{ => regression}/int_pruf_plant_analysis.py (100%) rename test/{ => regression}/int_turbine_long_term_gross_energy.py (100%) create mode 100644 test/unit/__init__.py rename test/{ => unit}/test_filter_toolkit.py (100%) rename test/{ => unit}/test_imputing_toolkit.py (100%) rename test/{ => unit}/test_met_data_processing_toolkit.py (100%) rename test/{ => unit}/test_ml_toolkit.py (100%) rename test/{ => unit}/test_power_curve_toolkit.py (100%) rename test/{ => unit}/test_timeseries_toolkit.py (100%) rename test/{ => unit}/test_unit_conversion_toolkit.py (100%) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 8ed9177c..5d3cd869 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -9,10 +9,11 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8] + os: [ubuntu-latest, windows-latest] + python-version: [3.6, 3.8] steps: - uses: actions/checkout@v2 @@ -24,11 +25,14 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + shell: bash - name: Unzip example data run: | - yes | unzip examples/data/la_haute_borne.zip -d examples/data/la_haute_borne/ + unzip examples/data/la_haute_borne.zip -d examples/data/la_haute_borne/ + shell: bash - name: Test with pytest run: | - pytest -o python_files=test/*.py --cov=operational_analysis --cov-report=xml + pytest --cov=operational_analysis --cov-report=xml + shell: bash - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e2162d9f..9414f876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog All notable changes to this project will be documented in this file. If you make a notable change to the project, please add a line describing the change to the "unreleased" section. The maintainers will make an effort to keep the [Github Releases](https://github.com/NREL/OpenOA/releases) page up to date with this changelog. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [2.0.1 - 2020-10-13] +- Replaced `GeoPandas` functionality with `pyproj` and `Shapely` for coordinate +reference system conversion and distance measurements. +- Moved and renamed tests and updated the documentation accordingly. + ## [2.0.0 - 2020-08-11] - Switch to [semantic versioning](https://semver.org) from this release forward. - Efficiency improvements in AEP calculation @@ -31,4 +36,4 @@ All notable changes to this project will be documented in this file. If you make - Turbine / Scada level toolkits: Filtering, Imputing, Met, Pandas Plotting, Timeseries, Unit Conversion - Most toolkits and all methods are fully documented in Sphinx. - Two example notebooks: Operational AEP Analysis and Turbine Analysis -- All toolkits except for Pandas Plotting have > 80% test coverage. \ No newline at end of file +- All toolkits except for Pandas Plotting have > 80% test coverage. diff --git a/examples/00_toolkit_examples.ipynb b/examples/00_toolkit_examples.ipynb index 41c0fbe0..a12164bf 100644 --- a/examples/00_toolkit_examples.ipynb +++ b/examples/00_toolkit_examples.ipynb @@ -60,7 +60,7 @@ } ], "source": [ - "project = Project_Engie('./data/la_haute_borne/')\n", + "project = Project_Engie('./data/la_haute_borne')\n", "project.prepare()" ] }, @@ -493,4 +493,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/02_plant_aep_analysis.ipynb b/examples/02_plant_aep_analysis.ipynb index 77e75a4a..20db40c7 100755 --- a/examples/02_plant_aep_analysis.ipynb +++ b/examples/02_plant_aep_analysis.ipynb @@ -76,7 +76,7 @@ "outputs": [], "source": [ "# Load plant object\n", - "project = Project_Engie('./data/la_haute_borne/')" + "project = Project_Engie('./data/la_haute_borne')" ] }, { @@ -1001,4 +1001,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/02b_augmented_plant_aep_analysis.ipynb b/examples/02b_augmented_plant_aep_analysis.ipynb index 9ea29bca..8b71894b 100644 --- a/examples/02b_augmented_plant_aep_analysis.ipynb +++ b/examples/02b_augmented_plant_aep_analysis.ipynb @@ -58,7 +58,7 @@ "outputs": [], "source": [ "# Load plant object\n", - "project = Project_Engie('./data/la_haute_borne/')" + "project = Project_Engie('./data/la_haute_borne')" ] }, { @@ -1240,7 +1240,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.4" }, "toc": { "base_numbering": 1, @@ -1258,4 +1258,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/examples/03_turbine_ideal_energy.ipynb b/examples/03_turbine_ideal_energy.ipynb index d0333c66..89de187b 100644 --- a/examples/03_turbine_ideal_energy.ipynb +++ b/examples/03_turbine_ideal_energy.ipynb @@ -62,7 +62,7 @@ "outputs": [], "source": [ "# Load plant object\n", - "project = Project_Engie('./data/la_haute_borne/')" + "project = Project_Engie('./data/la_haute_borne')" ] }, { @@ -628,7 +628,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.4" }, "toc": { "base_numbering": 1, @@ -646,4 +646,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/examples/04_electrical_losses.ipynb b/examples/04_electrical_losses.ipynb index 64944fea..975b41cf 100644 --- a/examples/04_electrical_losses.ipynb +++ b/examples/04_electrical_losses.ipynb @@ -71,7 +71,7 @@ "outputs": [], "source": [ "# Load wind farm object\n", - "project = Project_Engie('./data/la_haute_borne/')" + "project = Project_Engie('./data/la_haute_borne')" ] }, { @@ -426,4 +426,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/05_eya_gap_analysis.ipynb b/examples/05_eya_gap_analysis.ipynb index 5a745233..8519a78c 100644 --- a/examples/05_eya_gap_analysis.ipynb +++ b/examples/05_eya_gap_analysis.ipynb @@ -67,7 +67,7 @@ ], "source": [ "# Load plant object and process plant data\n", - "project = Project_Engie('./data/la_haute_borne/')\n", + "project = Project_Engie('./data/la_haute_borne')\n", "project.prepare()" ] }, @@ -252,4 +252,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/operational_analysis/__init__.py b/operational_analysis/__init__.py index e69f505c..d0bf0e0d 100644 --- a/operational_analysis/__init__.py +++ b/operational_analysis/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.0" +__version__ = "2.0.1" import json import logging diff --git a/operational_analysis/types/asset.py b/operational_analysis/types/asset.py index 45733198..1d593a60 100644 --- a/operational_analysis/types/asset.py +++ b/operational_analysis/types/asset.py @@ -1,17 +1,17 @@ import importlib import itertools -import geopandas as gp import numpy as np import pandas as pd +from pyproj import Transformer from shapely.geometry import Point class AssetData(object): """ - This class wraps around a GeoPandas dataframe that contains + This class wraps around a Pandas dataframe that contains metadata about the plant assets. It provides some useful functions - to work with this data (e.g., calculating nearest neighbors, etc.) + to work with this data (e.g., calculating nearest neighbors, etc.). """ def __init__(self, engine="pandas"): @@ -89,12 +89,10 @@ def parse_geometry(self, srs='epsg:4326', zone=None, longitude=None): longitude = self.df['longitude'].mean() zone = int(np.floor((180 + longitude) / 6.0)) + 1 - self._asset = gp.GeoDataFrame(self._asset) - self._asset['geometry'] = self._asset.apply(lambda x: Point(x['longitude'], x['latitude']), 1) - self._asset.set_geometry('geometry') - self._asset.crs = {'init': srs} - self._asset = self._asset.to_crs( - "+proj=utm +zone=" + str(zone) + " +ellps=WGS84 +datum=WGS84 +units=m +no_defs") + to_crs = f"+proj=utm +zone={zone} +ellps=WGS84 +datum=WGS84 +units=m +no_defs" + transformer = Transformer.from_crs(srs.upper(), to_crs) + lats, lons = transformer.transform(self._asset["latitude"].values, self._asset["longitude"].values) + self._asset["geometry"] = [Point(lat, lon) for lat, lon in zip (lats, lons)] def calculate_nearest(self, active_turbine_ids, active_tower_ids): """Create or overwrite a column called 'nearest_turbine_id' or 'nearest_tower_id' which contains the asset id @@ -124,11 +122,11 @@ def calculate_nearest(self, active_turbine_ids, active_tower_ids): def distance_matrix(self): ret = np.ones((self._asset.shape[0], self._asset.shape[0])) * -1 - for i, j in itertools.permutations(self._asset.index, 2): + for i, j in itertools.combinations(self._asset.index, 2): point1 = self._asset.loc[i, 'geometry'] point2 = self._asset.loc[j, 'geometry'] distance = point1.distance(point2) - ret[i][j] = distance + ret[i, j] = ret[j, i] = distance return ret def asset_ids(self): diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..edff37ab --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = test/*.py \ No newline at end of file diff --git a/readme.md b/readme.md index b2c6414e..dc08f6c8 100644 --- a/readme.md +++ b/readme.md @@ -31,19 +31,6 @@ conda create --name openoa-env python=3 conda activate openoa-env ``` -#### Special Note for users of Microsoft Windows: - -The Anaconda python distribution is *required* for users of Microsoft Windows. This is because the pip package of GDAL for Windows requires Visual Studio to compile some of the dependencies. While advanced users are welcome to explore this option, we find it is easier to install the following packages via Anaconda: - -``` -conda install shapely -conda install geos -conda install fiona -``` - -If errors about Visual Studio persist, you can try downloading the [Microsoft Visual Studio compiler for Python](https://www.microsoft.com/en-us/download/details.aspx?id=44266) and compiling GDAL yourself. - - ### Installation: Clone the repository and install the library and its dependencies using pip: @@ -88,13 +75,13 @@ unzip examples/data/la_haute_borne.zip -d examples/data/la_haute_borne/ Tests are written in the Python unittest framework and are runnable using pytest. To run all tests with code coverage reporting: ``` -pytest -o python_files=test/*.py --cov=operational_analysis +pytest --cov=operational_analysis ``` To run unit tests only: ``` -pytest -o python_files=test/test_*.py --cov=operational_analysis +pytest --ignore=test/regression/ --cov=operational_analysis ``` #### Documentation @@ -117,6 +104,7 @@ Nathan Agarwal, Nicola Bodini, Anna Craig, Jason Fields, +Rob Hammond, Travis Kemper, Joseph Lee, Monte Lunacek, diff --git a/requirements.txt b/requirements.txt index e0030cb1..ee3ff20f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Requirements for the OpenOA software package eia-python>=1.22 -geopandas>=0.4.0 +pyproj>=2.6.1 +shapely>=1.7.1 numpy>=1.15.4 pandas>=0.23.4 pygam>=0.8.0 diff --git a/setup.py b/setup.py index a736a2aa..7ad0a824 100644 --- a/setup.py +++ b/setup.py @@ -49,12 +49,13 @@ def read_file(filename): "scipy", "pandas", "pygam", - "geopandas", "tqdm", "statsmodels", "scikit_learn", "EIA-python", - "requests"], + "requests", + "pyproj", + "shapely"], tests_require=['pytest', 'pytest-cov'], python_requires='>=3.6' ) diff --git a/test/regression/__init__.py b/test/regression/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/int_electrical_losses.py b/test/regression/int_electrical_losses.py similarity index 100% rename from test/int_electrical_losses.py rename to test/regression/int_electrical_losses.py diff --git a/test/int_eya_gap_analysis.py b/test/regression/int_eya_gap_analysis.py similarity index 100% rename from test/int_eya_gap_analysis.py rename to test/regression/int_eya_gap_analysis.py diff --git a/test/int_pruf_plant_analysis.py b/test/regression/int_pruf_plant_analysis.py similarity index 100% rename from test/int_pruf_plant_analysis.py rename to test/regression/int_pruf_plant_analysis.py diff --git a/test/int_turbine_long_term_gross_energy.py b/test/regression/int_turbine_long_term_gross_energy.py similarity index 100% rename from test/int_turbine_long_term_gross_energy.py rename to test/regression/int_turbine_long_term_gross_energy.py diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_filter_toolkit.py b/test/unit/test_filter_toolkit.py similarity index 100% rename from test/test_filter_toolkit.py rename to test/unit/test_filter_toolkit.py diff --git a/test/test_imputing_toolkit.py b/test/unit/test_imputing_toolkit.py similarity index 100% rename from test/test_imputing_toolkit.py rename to test/unit/test_imputing_toolkit.py diff --git a/test/test_met_data_processing_toolkit.py b/test/unit/test_met_data_processing_toolkit.py similarity index 100% rename from test/test_met_data_processing_toolkit.py rename to test/unit/test_met_data_processing_toolkit.py diff --git a/test/test_ml_toolkit.py b/test/unit/test_ml_toolkit.py similarity index 100% rename from test/test_ml_toolkit.py rename to test/unit/test_ml_toolkit.py diff --git a/test/test_power_curve_toolkit.py b/test/unit/test_power_curve_toolkit.py similarity index 100% rename from test/test_power_curve_toolkit.py rename to test/unit/test_power_curve_toolkit.py diff --git a/test/test_timeseries_toolkit.py b/test/unit/test_timeseries_toolkit.py similarity index 100% rename from test/test_timeseries_toolkit.py rename to test/unit/test_timeseries_toolkit.py diff --git a/test/test_unit_conversion_toolkit.py b/test/unit/test_unit_conversion_toolkit.py similarity index 100% rename from test/test_unit_conversion_toolkit.py rename to test/unit/test_unit_conversion_toolkit.py