Meteostat 1.5.11 (#82)

* Bugfix and new endpoint path function  (#80)

* Fixes bug with chunked datasets in hourly requests and introduce a function to generate the endpoint path within a tested function

* Update linter.yml

Workaround for super-linter bug

Co-authored-by: Christian Lamprecht <[email protected]>

* Minor adaptions to better understand cache behaviour (#81)

* Restructuring & Linting

* misplaced-comparison-constant

* Wrap up v1.5.11

* Finalize 1.5.11

Co-authored-by: Daniel Lassahn <[email protected]>

.github/workflows/linter.yml

@@ -48,7 +48,7 @@ jobs:
       # Run Linter against code base #
       ################################
       - name: Lint Code Base
-        uses: github/super-linter@v3
+        uses: github/super-linter@v3.17.0
         env:
           VALIDATE_ALL_CODEBASE: false
           DEFAULT_BRANCH: master

meteostat/__init__.py

@@ -12,7 +12,7 @@
 """
 
 __appname__ = 'meteostat'
-__version__ = '1.5.10'
+__version__ = '1.5.11'
 
 from .interface.base import Base
 from .interface.timeseries import Timeseries

meteostat/core/cache.py

@@ -13,7 +13,7 @@
 import hashlib
 
 
-def get_file_path(
+def get_local_file_path(
     cache_dir: str,
     cache_subdir: str,
     path: str
@@ -25,8 +25,7 @@ def get_file_path(
     # Get file ID
     file = hashlib.md5(path.encode('utf-8')).hexdigest()
 
-    # Return path
-    return cache_dir + os.sep + cache_subdir + os.sep + file
+    return f"{cache_dir}/{cache_subdir}/{file}"
 
 
 def file_in_cache(

meteostat/core/loader.py

@@ -87,7 +87,8 @@ def load_handler(
             compression='gzip',
             names=columns,
             dtype=types,
-            parse_dates=parse_dates)
+            parse_dates=parse_dates
+        )
 
         # Force datetime conversion
         if coerce_dates:

meteostat/enumerations/granularity.py

@@ -0,0 +1,22 @@
+"""
+Granularity Enumeration
+
+Meteorological data provided by Meteostat (https://dev.meteostat.net)
+under the terms of the Creative Commons Attribution-NonCommercial
+4.0 International Public License.
+
+The code is licensed under the MIT license.
+"""
+
+from enum import Enum
+
+
+class Granularity(Enum):
+    """
+    The different levels of time series granularity
+    """
+
+    HOURLY = 'hourly'
+    DAILY = 'daily'
+    MONTHLY = 'monthly'
+    NORMALS = 'normals'

meteostat/interface/daily.py

@@ -12,10 +12,12 @@
 from typing import Union
 import numpy as np
 import pandas as pd
-from meteostat.core.cache import get_file_path, file_in_cache
+from meteostat.core.cache import get_local_file_path, file_in_cache
 from meteostat.core.loader import processing_handler, load_handler
+from meteostat.enumerations.granularity import Granularity
 from meteostat.utilities.validations import validate_series
 from meteostat.utilities.aggregations import degree_mean, weighted_average
+from meteostat.utilities.endpoint import generate_endpoint_path
 from meteostat.interface.timeseries import Timeseries
 from meteostat.interface.point import Point
 
@@ -93,11 +95,14 @@ def _load(
         """
 
         # File name
-        file = 'daily/' + ('full' if self._model else 'obs') + \
-            '/' + station + '.csv.gz'
+        file = generate_endpoint_path(
+            Granularity.DAILY,
+            station,
+            self._model
+        )
 
         # Get local file path
-        path = get_file_path(self.cache_dir, self.cache_subdir, file)
+        path = get_local_file_path(self.cache_dir, self.cache_subdir, file)
 
         # Check if file in cache
         if self.max_age > 0 and file_in_cache(path, self.max_age):
@@ -142,12 +147,7 @@ def _get_data(self) -> None:
         if len(self._stations) > 0:
 
             # List of datasets
-            datasets = []
-
-            for station in self._stations:
-                datasets.append((
-                    str(station),
-                ))
+            datasets = [(str(station),) for station in self._stations]
 
             # Data Processing
             return processing_handler(

meteostat/interface/hourly.py

@@ -9,15 +9,17 @@
 """
 
 from math import floor
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Union
 import pytz
 import numpy as np
 import pandas as pd
-from meteostat.core.cache import get_file_path, file_in_cache
+from meteostat.core.cache import get_local_file_path, file_in_cache
 from meteostat.core.loader import processing_handler, load_handler
+from meteostat.enumerations.granularity import Granularity
 from meteostat.utilities.validations import validate_series
 from meteostat.utilities.aggregations import degree_mean, weighted_average
+from meteostat.utilities.endpoint import generate_endpoint_path
 from meteostat.interface.timeseries import Timeseries
 from meteostat.interface.point import Point
 
@@ -32,7 +34,7 @@ class Hourly(Timeseries):
     # The cache subdirectory
     cache_subdir: str = 'hourly'
 
-    # Specify if the library should use chunks or full dumps
+    # Download data as annual chunks
     chunked: bool = True
 
     # The time zone
@@ -118,13 +120,15 @@ def _set_time(
 
                 # Set start date
                 self._start = timezone.localize(
-                    start, is_dst=None).astimezone(
-                    pytz.utc)
+                    start,
+                    is_dst=None
+                ).astimezone(pytz.utc)
 
                 # Set end date
                 self._end = timezone.localize(
-                    end, is_dst=None).astimezone(
-                    pytz.utc)
+                    end,
+                    is_dst=None
+                ).astimezone(pytz.utc)
 
         else:
 
@@ -134,21 +138,25 @@ def _set_time(
             # Set end date
             self._end = end
 
+        self._annual_steps = [
+            (
+                self._start + timedelta(days=365 * i)
+            ).year for i in range(
+                self._end.year - self._start.year + 1
+            )
+        ]
+
     def _load(
         self,
         station: str,
-        year: str = None
+        file: str
     ) -> None:
         """
         Load file from Meteostat
         """
 
-        # File name
-        file = 'hourly/' + ('full' if self._model else 'obs') + '/' + \
-            (year + '/' if year else '') + station + '.csv.gz'
-
         # Get local file path
-        path = get_file_path(self.cache_dir, self.cache_subdir, file)
+        path = get_local_file_path(self.cache_dir, self.cache_subdir, file)
 
         # Check if file in cache
         if self.max_age > 0 and file_in_cache(path, self.max_age):
@@ -164,7 +172,8 @@ def _load(
                 file,
                 self._columns,
                 self._types,
-                self._parse_dates)
+                self._parse_dates
+            )
 
             # Validate Series
             df = validate_series(df, station)
@@ -176,8 +185,12 @@ def _load(
         # Localize time column
         if self._timezone is not None and len(df.index) > 0:
             df = df.tz_localize(
-                'UTC', level='time').tz_convert(
-                self._timezone, level='time')
+                'UTC',
+                level='time'
+            ).tz_convert(
+                self._timezone,
+                level='time'
+            )
 
         # Filter time period and append to DataFrame
         if self._start and self._end:
@@ -198,29 +211,40 @@ def _get_data(self) -> None:
 
         if len(self._stations) > 0:
 
-            # List of datasets
-            datasets = []
-
-            for station in self._stations:
-
-                if self.chunked and self._start and self._end:
-
-                    for year in range(self._start.year, self._end.year + 1):
-                        datasets.append((
-                            str(station),
-                            str(year)
-                        ))
-
-                else:
+            # Create list of datasets
+            if self.chunked:
+                datasets = [
+                    (
+                        str(station),
+                        generate_endpoint_path(
+                            Granularity.HOURLY,
+                            station,
+                            self._model,
+                            year
+                        )
+                    )
+                    for station in self._stations for year in self._annual_steps
+                ]
 
-                    datasets.append((
+            else:
+                datasets = [
+                    (
                         str(station),
-                        None
-                    ))
+                        generate_endpoint_path(
+                            Granularity.HOURLY,
+                            station,
+                            self._model
+                        )
+                    )
+                    for station in self._stations
+                ]
 
-            # Data Processing
             return processing_handler(
-                datasets, self._load, self.processes, self.threads)
+                datasets,
+                self._load,
+                self.processes,
+                self.threads
+            )
 
         return pd.DataFrame(columns=[*self._types])
 

meteostat/interface/monthly.py

@@ -12,10 +12,12 @@
 from typing import Union
 import numpy as np
 import pandas as pd
-from meteostat.core.cache import get_file_path, file_in_cache
+from meteostat.core.cache import get_local_file_path, file_in_cache
 from meteostat.core.loader import processing_handler, load_handler
+from meteostat.utilities.endpoint import generate_endpoint_path
 from meteostat.utilities.validations import validate_series
 from meteostat.utilities.aggregations import degree_mean, weighted_average
+from meteostat.enumerations.granularity import Granularity
 from meteostat.interface.timeseries import Timeseries
 from meteostat.interface.point import Point
 
@@ -94,11 +96,14 @@ def _load(
         """
 
         # File name
-        file = 'monthly/' + ('full' if self._model else 'obs') + \
-            '/' + station + '.csv.gz'
+        file = generate_endpoint_path(
+            Granularity.MONTHLY,
+            station,
+            self._model
+        )
 
         # Get local file path
-        path = get_file_path(self.cache_dir, self.cache_subdir, file)
+        path = get_local_file_path(self.cache_dir, self.cache_subdir, file)
 
         # Check if file in cache
         if self.max_age > 0 and file_in_cache(path, self.max_age):
@@ -143,15 +148,11 @@ def _get_data(self) -> None:
         if len(self._stations) > 0:
 
             # List of datasets
-            datasets = []
-
-            for station in self._stations:
-                datasets.append((
-                    str(station),
-                ))
-
+            datasets = [(str(station),) for station in self._stations]
             # Data Processing
-            return processing_handler(datasets, self._load, self.processes, self.threads)
+            return processing_handler(
+                datasets, self._load, self.processes, self.threads
+            )
 
         # Empty DataFrame
         return pd.DataFrame(columns=[*self._types])

meteostat/interface/normals.py

@@ -13,10 +13,12 @@
 from datetime import datetime
 import numpy as np
 import pandas as pd
-from meteostat.core.cache import get_file_path, file_in_cache
+from meteostat.core.cache import get_local_file_path, file_in_cache
+from meteostat.enumerations.granularity import Granularity
 from meteostat.core.loader import processing_handler, load_handler
 from meteostat.core.warn import warn
 from meteostat.utilities.aggregations import weighted_average
+from meteostat.utilities.endpoint import generate_endpoint_path
 from meteostat.interface.base import Base
 from meteostat.interface.point import Point
 
@@ -78,10 +80,13 @@ def _load(
         """
 
         # File name
-        file = f'normals/{station}.csv.gz'
+        file = generate_endpoint_path(
+            Granularity.NORMALS,
+            station
+        )
 
         # Get local file path
-        path = get_file_path(self.cache_dir, self.cache_subdir, file)
+        path = get_local_file_path(self.cache_dir, self.cache_subdir, file)
 
         # Check if file in cache
         if self.max_age > 0 and file_in_cache(path, self.max_age):
@@ -97,11 +102,13 @@ def _load(
                 file,
                 self._columns,
                 self._types,
-                None)
+                None
+            )
 
             if df.index.size > 0:
 
                 # Add weather station ID
+                # pylint: disable=unsupported-assignment-operation
                 df['station'] = station
 
                 # Set index
@@ -130,12 +137,7 @@ def _get_data(self) -> None:
         if len(self._stations) > 0:
 
             # List of datasets
-            datasets = []
-
-            for station in self._stations:
-                datasets.append((
-                    str(station),
-                ))
+            datasets = [(str(station),) for station in self._stations]
 
             # Data Processing
             return processing_handler(