Remove network calls from tests (#205)

* Remove network calls from tests

This update removes network calls wherever they exist, and also adds new unit tests for functionality related to API calls.

* Clean up

.github/workflows/ci.yml

@@ -30,11 +30,14 @@ jobs:
       - name: Create env file
         run: |
           touch .env
-          echo NREL_API_KEY=${{ secrets.NREL_API_KEY }} >> .env
-          cat .env
-      - name: Unit tests
+      #     echo NREL_API_KEY=${{ secrets.NREL_API_KEY }} >> .env
+      #     cat .env
+      - name: Install test dependencies
         run: |
           pip install pytest
+          pip install responses
+      - name: Run tests
+        run: |
           PYTHONPATH=. pytest tests
       - name: Lint with flake8
         run: |

hopp/__init__.py

@@ -4,4 +4,7 @@
 with open(Path(__file__).parent / "version.py") as _version_file:
     __version__ = _version_file.read().strip()
 
-ROOT_DIR = Path(__file__).resolve().parent
+ROOT_DIR = Path(__file__).resolve().parent
+
+# environment variable used to specify a testing environment
+TEST_ENV_VAR="TEST"

hopp/simulation/technologies/layout/flicker_mismatch.py

@@ -1,10 +1,11 @@
 from typing import List, Union, Optional, Sequence
-import multiprocessing_on_dill as mp
 from pathlib import Path
-import functools
 import copy
 from itertools import product
 import sys
+
+import multiprocessing_on_dill as mp
+import functools
 import matplotlib.pyplot as plt
 sys.path.append('.')
 
@@ -13,6 +14,7 @@
 from pvmismatch import pvsystem
 import PySAM.Pvwattsv8 as pv
 
+from hopp import ROOT_DIR
 from hopp.utilities.log import flicker_logger as logger
 from hopp.simulation.technologies.resource import SolarResource
 from hopp.simulation.technologies.layout.shadow_flicker import get_sun_pos, get_turbine_shadows_timeseries, create_pv_string_points
@@ -167,7 +169,7 @@ def _setup_irradiance(self):
         pv_model.SystemDesign.gcr = .1
         if self.solar_resource_data is None:
             filename = str(self.lat) + "_" + str(self.lon) + "_psmv3_60_2012.csv"
-            weather_path = Path(__file__).parent.parent.parent / "resource_files" / "solar" / filename
+            weather_path = ROOT_DIR.parent / "resource_files" / "solar" / filename
             if not weather_path.is_file():
                 SolarResource(self.lat, self.lon, year=2012)
                 if not weather_path.is_file():

hopp/simulation/technologies/reopt.py

@@ -16,6 +16,8 @@
 import urllib3
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+# enable to record responses for testing/debugging
+# from responses import _recorder
 
 class REopt:
     """
@@ -245,7 +247,7 @@ def create_post(self, solar_model: PVPlant, wind_model: WindPlant, batt_model: B
         # logger.info("Created REopt post, exported to " + post_path)
         return post
 
-    def get_reopt_results(self, results_file=None):
+    def get_reopt_results(self, results_file=None, poll_interval=5):
         """
         Function for posting job and polling results end-point
         :param post:
@@ -261,7 +263,7 @@ def get_reopt_results(self, results_file=None):
 
         if run_id is not None:
             results_url = self.reopt_api_url + '<run_uuid>/results/?api_key=' + self.api_key
-            results = self.poller(url=results_url.replace('<run_uuid>', run_id))
+            results = self.poller(results_url.replace('<run_uuid>', run_id), poll_interval)
 
             with open(results_file, 'w') as fp:
                 json.dump(obj=results, fp=fp)
@@ -274,6 +276,8 @@ def get_reopt_results(self, results_file=None):
         return results
 
     @staticmethod
+    # enable to record the response for testing/debugging
+    # @_recorder.record(file_path="reopt_responses.yaml")
     def poller(url, poll_interval=5):
         """
         Function for polling the REopt API results URL until status is not "Optimizing..."
@@ -317,6 +321,8 @@ def poller(url, poll_interval=5):
         return resp_dict
 
     @staticmethod
+    # enable to record the response for testing/debugging
+    # @_recorder.record(file_path="reopt_responses_post.yaml")
     def get_run_uuid(post, API_KEY, api_url):
         """
         Function for posting job

hopp/simulation/technologies/resource/solar_resource.py

@@ -9,6 +9,9 @@
 from hopp.simulation.technologies.resource.resource import Resource
 
 
+BASE_URL = "https://developer.nrel.gov/api/nsrdb/v2/solar/psm3-download.csv"
+
+
 class SolarResource(Resource):
     """
         Class to manage Solar Resource data
@@ -53,8 +56,8 @@ def __init__(self, lat, lon, year, path_resource="", filepath="", **kwargs):
         logger.info("SolarResource: {}".format(self.filename))
 
     def download_resource(self):
-        url = 'https://developer.nrel.gov/api/nsrdb/v2/solar/psm3-download.csv?wkt=POINT({lon}+{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(
-            year=self.year, lat=self.latitude, lon=self.longitude, leap=self.leap_year, interval=self.interval,
+        url = '{base}?wkt=POINT({lon}+{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(
+            base=BASE_URL, year=self.year, lat=self.latitude, lon=self.longitude, leap=self.leap_year, interval=self.interval,
             utc=self.utc, name=self.name, email=self.email,
             mailing_list=self.mailing_list, affiliation=self.affiliation, reason=self.reason, api=get_developer_nrel_gov_key(),
             attr=self.solar_attributes)

hopp/simulation/technologies/resource/wind_resource.py

@@ -1,12 +1,13 @@
 import csv, os
-
 from PySAM.ResourceTools import SRW_to_wind_data
 
 from hopp.utilities.keys import get_developer_nrel_gov_key
-from hopp.utilities.log import hybrid_logger as logger
 from hopp.simulation.technologies.resource.resource import Resource
 
 
+BASE_URL = "https://developer.nrel.gov/api/wind-toolkit/v2/wind/wtk-srw-download"
+
+
 class WindResource(Resource):
     """ Class to manage Wind Resource data
 
@@ -98,8 +99,8 @@ def download_resource(self):
         if not success:
 
             for height, f in self.file_resource_heights.items():
-                url = 'https://developer.nrel.gov/api/wind-toolkit/v2/wind/wtk-srw-download?year={year}&lat={lat}&lon={lon}&hubheight={hubheight}&api_key={api_key}&email={email}'.format(
-                    year=self.year, lat=self.latitude, lon=self.longitude, hubheight=height, api_key=get_developer_nrel_gov_key(), email=self.email)
+                url = '{base}?year={year}&lat={lat}&lon={lon}&hubheight={hubheight}&api_key={api_key}&email={email}'.format(
+                    base=BASE_URL, year=self.year, lat=self.latitude, lon=self.longitude, hubheight=height, api_key=get_developer_nrel_gov_key(), email=self.email)
 
                 success = self.call_api(url, filename=f)
 

hopp/simulation/technologies/utility_rate.py

@@ -4,6 +4,7 @@
 
 from hopp.utilities.keys import get_developer_nrel_gov_key
 
+URDB_BASE_URL = "https://api.openei.org/utility_rates"
 
 class UtilityRate:
     """
@@ -13,6 +14,11 @@ class UtilityRate:
     def __init__(self, path_rates, urdb_label):
         self.path_rates = path_rates
         self.urdb_label = urdb_label
+        self.api_key = get_developer_nrel_gov_key()
+
+    @property
+    def urdb_url(self):
+        return f"{URDB_BASE_URL}?version=7&format=json&detail=full&getpage={self.urdb_label}&api_key={self.api_key}"
 
     def get_urdb_response(self):
         file_exists = False
@@ -21,11 +27,8 @@ def get_urdb_response(self):
             file_exists = os.path.exists(file_urdb_json)
         results = None
         if not file_exists and self.urdb_label is not None:
-            urdb_url = 'https://api.openei.org/utility_rates?version=7&format=json&detail=full&getpage={urdb_label}&api_key={api_key}'.format(
-                urdb_label=self.urdb_label, api_key=get_developer_nrel_gov_key())
-
             # since NREL can't figure out its certificate
-            resp = requests.get(urdb_url, verify=False)
+            resp = requests.get(self.urdb_url, verify=False)
 
             if resp.ok:
                 results = json.loads(resp.text, strict=False)
@@ -34,7 +37,7 @@ def get_urdb_response(self):
                     with open(file_urdb_json, 'w') as fp:
                         json.dump(obj=results, fp=fp)
                 self.urdb_response = results
-        elif os.path.exists(file_urdb_json):
+        elif file_exists:
             with open(file_urdb_json, 'r') as fp:
                 results = json.load(fp=fp)
         self.results = results

requirements-dev.txt

@@ -16,6 +16,7 @@ pytest
 python-dotenv
 pytz
 requests
+responses
 scipy
 timezonefinder