diff --git a/CHANGELOG.md b/CHANGELOG.md index 7103426dd..4a8ea6f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added +- ENH: Exponential backoff decorator (fix #449) [#588](https://github.com/RocketPy-Team/RocketPy/pull/588) - ENH: Add new stability margin properties to Flight class [#572](https://github.com/RocketPy-Team/RocketPy/pull/572) - ENH: adds `Function.remove_outliers` method [#554](https://github.com/RocketPy-Team/RocketPy/pull/554) diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index b72b2cb38..5a845a173 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -3,7 +3,7 @@ import re import warnings from collections import namedtuple -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import numpy as np import numpy.ma as ma @@ -13,6 +13,7 @@ from ..mathutils.function import Function, funcify_method from ..plots.environment_plots import _EnvironmentPlots from ..prints.environment_prints import _EnvironmentPrints +from ..tools import exponential_backoff try: import netCDF4 @@ -680,18 +681,9 @@ def set_elevation(self, elevation="Open-Elevation"): # self.elevation = elev - elif self.latitude != None and self.longitude != None: - try: - print("Fetching elevation from open-elevation.com...") - request_url = "https://api.open-elevation.com/api/v1/lookup?locations={:f},{:f}".format( - self.latitude, self.longitude - ) - response = requests.get(request_url) - results = response.json()["results"] - self.elevation = results[0]["elevation"] - print("Elevation received:", self.elevation) - except: - raise RuntimeError("Unable to reach Open-Elevation API servers.") + elif self.latitude is not None and self.longitude is not None: + self.elevation = self.__fetch_open_elevation() + print("Elevation received: ", self.elevation) else: raise ValueError( "Latitude and longitude must be set to use" @@ -1303,26 +1295,8 @@ def set_atmospheric_model( "v_wind": "vgrdprs", } # Attempt to get latest forecast - time_attempt = datetime.utcnow() - success = False - attempt_count = 0 - while not success and attempt_count < 10: - time_attempt -= timedelta(hours=6 * attempt_count) - file = "https://nomads.ncep.noaa.gov/dods/gens_bc/gens{:04d}{:02d}{:02d}/gep_all_{:02d}z".format( - time_attempt.year, - time_attempt.month, - time_attempt.day, - 6 * (time_attempt.hour // 6), - ) - try: - self.process_ensemble(file, dictionary) - success = True - except OSError: - attempt_count += 1 - if not success: - raise RuntimeError( - "Unable to load latest weather data for GEFS through " + file - ) + self.__fetch_gefs_ensemble(dictionary) + elif file == "CMC": # Define dictionary dictionary = { @@ -1338,27 +1312,7 @@ def set_atmospheric_model( "u_wind": "ugrdprs", "v_wind": "vgrdprs", } - # Attempt to get latest forecast - time_attempt = datetime.utcnow() - success = False - attempt_count = 0 - while not success and attempt_count < 10: - time_attempt -= timedelta(hours=12 * attempt_count) - file = "https://nomads.ncep.noaa.gov/dods/cmcens/cmcens{:04d}{:02d}{:02d}/cmcens_all_{:02d}z".format( - time_attempt.year, - time_attempt.month, - time_attempt.day, - 12 * (time_attempt.hour // 12), - ) - try: - self.process_ensemble(file, dictionary) - success = True - except OSError: - attempt_count += 1 - if not success: - raise RuntimeError( - "Unable to load latest weather data for CMC through " + file - ) + self.__fetch_cmc_ensemble(dictionary) # Process other forecasts or reanalysis else: # Check if default dictionary was requested @@ -1650,20 +1604,7 @@ def process_windy_atmosphere(self, model="ECMWF"): model. """ - # Process the model string - model = model.lower() - if model[-1] == "u": # case iconEu - model = "".join([model[:4], model[4].upper(), model[4 + 1 :]]) - # Load data from Windy.com: json file - url = f"https://node.windy.com/forecast/meteogram/{model}/{self.latitude}/{self.longitude}/?step=undefined" - try: - response = requests.get(url).json() - except: - if model == "iconEu": - raise ValueError( - "Could not get a valid response for Icon-EU from Windy. Check if the latitude and longitude coordinates set are inside Europe.", - ) - raise + response = self.__fetch_atmospheric_data_from_windy(model) # Determine time index from model time_array = np.array(response["data"]["hours"]) @@ -1824,18 +1765,7 @@ def process_wyoming_sounding(self, file): None """ # Request Wyoming Sounding from file url - response = requests.get(file) - if response.status_code != 200: - raise ImportError("Unable to load " + file + ".") - if len(re.findall("Can't get .+ Observations at", response.text)): - raise ValueError( - re.findall("Can't get .+ Observations at .+", response.text)[0] - + " Check station number and date." - ) - if response.text == "Invalid OUTPUT: specified\n": - raise ValueError( - "Invalid OUTPUT: specified. Make sure the output is Text: List." - ) + response = self.__fetch_wyoming_sounding(file) # Process Wyoming Sounding by finding data table and station info response_split_text = re.split("(<.{0,1}PRE>)", response.text) @@ -1961,9 +1891,7 @@ def process_noaaruc_sounding(self, file): None """ # Request NOAA Ruc Sounding from file url - response = requests.get(file) - if response.status_code != 200 or len(response.text) < 10: - raise ImportError("Unable to load " + file + ".") + response = self.__fetch_noaaruc_sounding(file) # Split response into lines lines = response.text.split("\n") @@ -3538,6 +3466,110 @@ def set_earth_geometry(self, datum): f"The reference system {datum} for Earth geometry " "is not recognized." ) + # Auxiliary functions - Fetching Data from 3rd party APIs + + @exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) + def __fetch_open_elevation(self): + print("Fetching elevation from open-elevation.com...") + request_url = ( + "https://api.open-elevation.com/api/v1/lookup?locations" + f"={self.latitude},{self.longitude}" + ) + try: + response = requests.get(request_url) + except Exception as e: + raise RuntimeError("Unable to reach Open-Elevation API servers.") + results = response.json()["results"] + return results[0]["elevation"] + + @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) + def __fetch_atmospheric_data_from_windy(self, model): + model = model.lower() + if model[-1] == "u": # case iconEu + model = "".join([model[:4], model[4].upper(), model[4 + 1 :]]) + url = ( + f"https://node.windy.com/forecast/meteogram/{model}/" + f"{self.latitude}/{self.longitude}/?step=undefined" + ) + try: + response = requests.get(url).json() + except Exception as e: + if model == "iconEu": + raise ValueError( + "Could not get a valid response for Icon-EU from Windy. " + "Check if the coordinates are set inside Europe." + ) + return response + + @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) + def __fetch_wyoming_sounding(self, file): + response = requests.get(file) + if response.status_code != 200: + raise ImportError(f"Unable to load {file}.") + if len(re.findall("Can't get .+ Observations at", response.text)): + raise ValueError( + re.findall("Can't get .+ Observations at .+", response.text)[0] + + " Check station number and date." + ) + if response.text == "Invalid OUTPUT: specified\n": + raise ValueError( + "Invalid OUTPUT: specified. Make sure the output is Text: List." + ) + return response + + @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) + def __fetch_noaaruc_sounding(self, file): + response = requests.get(file) + if response.status_code != 200 or len(response.text) < 10: + raise ImportError("Unable to load " + file + ".") + + @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) + def __fetch_gefs_ensemble(self, dictionary): + time_attempt = datetime.now(tz=timezone.utc) + success = False + attempt_count = 0 + while not success and attempt_count < 10: + time_attempt -= timedelta(hours=6 * attempt_count) + file = ( + f"https://nomads.ncep.noaa.gov/dods/gens_bc/gens" + f"{time_attempt.year:04d}{time_attempt.month:02d}" + f"{time_attempt.day:02d}/" + f"gep_all_{6 * (time_attempt.hour // 6):02d}z" + ) + try: + self.process_ensemble(file, dictionary) + success = True + except OSError: + attempt_count += 1 + if not success: + raise RuntimeError( + "Unable to load latest weather data for GEFS through " + file + ) + + @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) + def __fetch_cmc_ensemble(self, dictionary): + # Attempt to get latest forecast + time_attempt = datetime.now(tz=timezone.utc) + success = False + attempt_count = 0 + while not success and attempt_count < 10: + time_attempt -= timedelta(hours=12 * attempt_count) + file = ( + f"https://nomads.ncep.noaa.gov/dods/cmcens/" + f"cmcens{time_attempt.year:04d}{time_attempt.month:02d}" + f"{time_attempt.day:02d}/" + f"cmcens_all_{12 * (time_attempt.hour // 12):02d}z" + ) + try: + self.process_ensemble(file, dictionary) + success = True + except OSError: + attempt_count += 1 + if not success: + raise RuntimeError( + "Unable to load latest weather data for CMC through " + file + ) + # Auxiliary functions - Geodesic Coordinates @staticmethod diff --git a/rocketpy/tools.py b/rocketpy/tools.py index acd0f4c27..ccecffd4f 100644 --- a/rocketpy/tools.py +++ b/rocketpy/tools.py @@ -1,6 +1,8 @@ +import functools import importlib import importlib.metadata import re +import time from bisect import bisect_left import numpy as np @@ -356,6 +358,25 @@ def check_requirement_version(module_name, version): return True +def exponential_backoff(max_attempts, base_delay=1, max_delay=60): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + delay = base_delay + for i in range(max_attempts): + try: + return func(*args, **kwargs) + except Exception as e: + if i == max_attempts - 1: + raise e from None + delay = min(delay * 2, max_delay) + time.sleep(delay) + + return wrapper + + return decorator + + def parallel_axis_theorem_from_com(com_inertia_moment, mass, distance): """Calculates the moment of inertia of a object relative to a new axis using the parallel axis theorem. The new axis is parallel to and at a distance diff --git a/tests/fixtures/environment/environment_fixtures.py b/tests/fixtures/environment/environment_fixtures.py index 8949f9973..851be3203 100644 --- a/tests/fixtures/environment/environment_fixtures.py +++ b/tests/fixtures/environment/environment_fixtures.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta import pytest + from rocketpy import Environment, EnvironmentAnalysis @@ -54,8 +55,8 @@ def env_analysis(): EnvironmentAnalysis """ env_analysis = EnvironmentAnalysis( - start_date=datetime.datetime(2019, 10, 23), - end_date=datetime.datetime(2021, 10, 23), + start_date=datetime(2019, 10, 23), + end_date=datetime(2021, 10, 23), latitude=39.3897, longitude=-8.28896388889, start_hour=6, diff --git a/tests/test_environment.py b/tests/test_environment.py index 5fa0e2c45..7349d512b 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,5 +1,4 @@ import datetime -import time from unittest.mock import patch import pytest @@ -64,13 +63,8 @@ def test_wyoming_sounding_atmosphere(mock_show, example_plain_env): # "file" option, instead of receiving the URL as a string. URL = "http://weather.uwyo.edu/cgi-bin/sounding?region=samer&TYPE=TEXT%3ALIST&YEAR=2019&MONTH=02&FROM=0500&TO=0512&STNM=83779" # give it at least 5 times to try to download the file - for i in range(5): - try: - example_plain_env.set_atmospheric_model(type="wyoming_sounding", file=URL) - break - except: - time.sleep(1) # wait 1 second before trying again - pass + example_plain_env.set_atmospheric_model(type="wyoming_sounding", file=URL) + assert example_plain_env.all_info() == None assert abs(example_plain_env.pressure(0) - 93600.0) < 1e-8 assert ( diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index ac25533eb..8d676f426 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -1,10 +1,10 @@ +import json import os import numpy as np import numpy.ma as ma import pytest import pytz -import json from rocketpy import Environment