From c503b4cb076c4e4d20bc79bbe7a13b6bfc39fb59 Mon Sep 17 00:00:00 2001 From: Franz-Benjamin Mocnik Date: Mon, 16 Aug 2021 05:55:44 +0200 Subject: [PATCH] cacheObject: caching strategies introduced IV --- OSMPythonTools/cachingStrategy/__init__.py | 5 ++-- OSMPythonTools/cachingStrategy/base.py | 12 ++++++++ OSMPythonTools/cachingStrategy/json.py | 21 ++++---------- OSMPythonTools/cachingStrategy/pickle.py | 32 ++++++---------------- OSMPythonTools/cachingStrategy/strategy.py | 25 +++++++++-------- OSMPythonTools/internal/cacheObject.py | 9 +++--- docs/general-remarks.md | 18 ++++++------ tests/test_cache.py | 14 +++++----- 8 files changed, 63 insertions(+), 73 deletions(-) create mode 100644 OSMPythonTools/cachingStrategy/base.py diff --git a/OSMPythonTools/cachingStrategy/__init__.py b/OSMPythonTools/cachingStrategy/__init__.py index dc64b8c..05daf91 100644 --- a/OSMPythonTools/cachingStrategy/__init__.py +++ b/OSMPythonTools/cachingStrategy/__init__.py @@ -1,2 +1,3 @@ -from OSMPythonTools.cachingStrategy.json import CachingStrategyJSON -from OSMPythonTools.cachingStrategy.pickle import CachingStrategyPickle +from OSMPythonTools.cachingStrategy.json import JSON +from OSMPythonTools.cachingStrategy.pickle import Pickle +from OSMPythonTools.cachingStrategy.strategy import CachingStrategy diff --git a/OSMPythonTools/cachingStrategy/base.py b/OSMPythonTools/cachingStrategy/base.py new file mode 100644 index 0000000..b836bbe --- /dev/null +++ b/OSMPythonTools/cachingStrategy/base.py @@ -0,0 +1,12 @@ +class CachingStrategyBase: + def __init__(self): + pass + + def get(self, key): + raise(NotImplementedError('Subclass should implement get')) + + def set(self, key, data): + raise(NotImplementedError('Subclass should implement set')) + + def close(self): + pass diff --git a/OSMPythonTools/cachingStrategy/json.py b/OSMPythonTools/cachingStrategy/json.py index 0cd1e80..b7c264b 100644 --- a/OSMPythonTools/cachingStrategy/json.py +++ b/OSMPythonTools/cachingStrategy/json.py @@ -1,22 +1,11 @@ import os.path import ujson -from OSMPythonTools.cachingStrategy.strategy import CachingStrategy - -class CachingStrategyJSON(CachingStrategy): - _instance = None +from OSMPythonTools.cachingStrategy.base import CachingStrategyBase +class JSON(CachingStrategyBase): def __init__(self, cacheDir='cache'): - raise RuntimeError('Call instance() instead') - - @classmethod - def instance(cls, cacheDir='cache'): - if cls._instance is None: - cls._instance = cls.__new__(cls) - else: - cls._instance.close() - cls._instance._cacheDir = cacheDir - return cls._instance + self._cacheDir = cacheDir def _filename(self, key): return os.path.join(self._cacheDir, key) @@ -31,6 +20,6 @@ def get(self, key): data = ujson.load(file) return data - def set(self, key, data): + def set(self, key, value): with open(self._filename(key), 'w') as file: - ujson.dump(data, file) + ujson.dump(value, file) diff --git a/OSMPythonTools/cachingStrategy/pickle.py b/OSMPythonTools/cachingStrategy/pickle.py index 3ad62b3..b05f2c2 100644 --- a/OSMPythonTools/cachingStrategy/pickle.py +++ b/OSMPythonTools/cachingStrategy/pickle.py @@ -2,41 +2,25 @@ import pickle import gzip as libraryGzip -from OSMPythonTools.cachingStrategy.strategy import CachingStrategy +from OSMPythonTools.cachingStrategy.base import CachingStrategyBase -class CachingStrategyPickle(CachingStrategy): - _instance = None - - def __init__(self, cacheDir='cache'): - raise RuntimeError('Call instance() instead') - - @classmethod - def instance(cls, cacheFile='cache', gzip=True): - if cls._instance is None: - cls._instance = cls.__new__(cls) - cls._instance.close() - cls._instance._cacheFileRaw = cacheFile - cls._instance.useGzip(gzip) - return cls._instance - - def useGzip(self, gzip=True): - if self._cache is not None: - self.close() - self._cacheFile = self._cacheFileRaw + '.pickle' + ('.gzip' if gzip else '') +class Pickle(CachingStrategyBase): + def __init__(self, cacheFile='cache', gzip=True): + self._cacheFile = cacheFile + '.pickle' + ('.gzip' if gzip else '') self._open = libraryGzip.open if gzip else open - return self + self.close() def get(self, key): if self._cache is None: self.open() return self._cache[key] if key in self._cache else None - def set(self, key, data): + def set(self, key, value): if self._cache is None: self.open() with self._open(self._cacheFile, 'wb') as file: - pickle.dump((key, data), file) - self._cache[key] = data + pickle.dump((key, value), file) + self._cache[key] = value def open(self): if os.path.exists(self._cacheFile): diff --git a/OSMPythonTools/cachingStrategy/strategy.py b/OSMPythonTools/cachingStrategy/strategy.py index c90b3cd..37f0bd5 100644 --- a/OSMPythonTools/cachingStrategy/strategy.py +++ b/OSMPythonTools/cachingStrategy/strategy.py @@ -1,12 +1,15 @@ -class CachingStrategy: - def __init__(self): - pass +from OSMPythonTools.cachingStrategy import JSON - def get(self, key): - raise(NotImplementedError('Subclass should implement get')) - - def set(self, key, data): - raise(NotImplementedError('Subclass should implement set')) - - def close(self): - pass +class CachingStrategy(): + __strategy = JSON() + @classmethod + def use(cls, strategy, **kwargs): + cls.__strategy.close() + cls.__strategy = strategy(**kwargs) + return cls.__strategy + @classmethod + def get(cls, key): + return cls.__strategy.get(key) + @classmethod + def set(cls, key, value): + cls.__strategy.set(key, value) diff --git a/OSMPythonTools/internal/cacheObject.py b/OSMPythonTools/internal/cacheObject.py index 30ba142..4d0b1b0 100755 --- a/OSMPythonTools/internal/cacheObject.py +++ b/OSMPythonTools/internal/cacheObject.py @@ -6,13 +6,12 @@ import urllib.request import OSMPythonTools -from OSMPythonTools.cachingStrategy.json import CachingStrategyJSON +from OSMPythonTools.cachingStrategy import CachingStrategy class CacheObject: - def __init__(self, prefix, endpoint, cachingStrategy=CachingStrategyJSON.instance(), waitBetweenQueries=None, jsonResult=True, userAgent=None): + def __init__(self, prefix, endpoint, waitBetweenQueries=None, jsonResult=True, userAgent=None): self._prefix = prefix self._endpoint = endpoint - self.__cachingStrategy = cachingStrategy self.__waitBetweenQueries = waitBetweenQueries self.__lastQuery = None self.__jsonResult = jsonResult @@ -21,7 +20,7 @@ def __init__(self, prefix, endpoint, cachingStrategy=CachingStrategyJSON.instanc def query(self, *args, onlyCached=False, shallow=False, **kwargs): queryString, hashString, params = self._queryString(*args, **kwargs) key = self._prefix + '-' + self.__hash(hashString + ('????' + urllib.parse.urlencode(sorted(params.items())) if params else '')) - data = self.__cachingStrategy.get(key) + data = CachingStrategy.get(key) makeDownload = False if data is not None: if 'version' not in data or 'response' not in data or 'timestamp' not in data: @@ -54,7 +53,7 @@ def query(self, *args, onlyCached=False, shallow=False, **kwargs): OSMPythonTools.logger.exception(msg) raise(Exception(msg)) if makeDownload: - self.__cachingStrategy.set(key, data) + CachingStrategy.set(key, data) return result def deleteQueryFromCache(self, *args, **kwargs): diff --git a/docs/general-remarks.md b/docs/general-remarks.md index 3a38b8d..d9f8a67 100644 --- a/docs/general-remarks.md +++ b/docs/general-remarks.md @@ -67,17 +67,19 @@ Please note that the default part cannot and should not be removed from the user ### Caching Strategies -The data is cached to ensure that the resources of the various services employed by this library, including the [Overpass endpoint](https://wiki.openstreetmap.org/wiki/Overpass_API), [Nominatim](http://nominatim.openstreetmap.org), and the [OSM API](https://wiki.openstreetmap.org/wiki/API), are not overused. While the caching strategies supported are very similar in structure, they store the data in different formats. By default, the data is stored in individual files in the JSON format: +The data is cached to ensure that the resources of the various services employed by this library, including the [Overpass endpoint](https://wiki.openstreetmap.org/wiki/Overpass_API), [Nominatim](http://nominatim.openstreetmap.org), and the [OSM API](https://wiki.openstreetmap.org/wiki/API), are not overused. While the caching strategies supported are very similar in structure, they store the data in different formats. By default, the data is stored in individual files in the JSON format. As an alternative, the data can also be [pickled](https://docs.python.org/3/library/pickle.html) and stored in one file: ```python -from OSMPythonTools.cachingStrategy import CachingStrategyJSON, CachingStrategyPickle -api = Api() -api = Api(cachingStrategy=CachingStrategyJSON()) # this is the default +from OSMPythonTools.cachingStrategy import CachingStrategy, JSON, Pickle +CachingStrategy.use(Pickle) +... ``` -As an alternative, the data can also be [pickled](https://docs.python.org/3/library/pickle.html) and stored in one file: +The caching strategy chosen applies to all subsequent requests. The pickle file is gzipped by default. This behaviour can, however, be changed: ```python -api = Api(cachingStrategy=CachingStrategyPickle()) +CachingStrategy.use(Pickle, gzip=False) +... ``` -The pickle file is gzipped by default, the latter of which can be disabled: +The default behaviourcan be restored as follows: ```python -api = Api(cachingStrategy=CachingStrategyPickle(gzip=False)) +CachingStrategy.use(JSON) +... ``` diff --git a/tests/test_cache.py b/tests/test_cache.py index 9e46c8d..5f85156 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,24 +1,24 @@ import pytest -from OSMPythonTools import cachingStrategy from OSMPythonTools.api import Api -from OSMPythonTools.cachingStrategy import CachingStrategyJSON, CachingStrategyPickle +from OSMPythonTools.cachingStrategy import CachingStrategy, JSON, Pickle @pytest.mark.parametrize(('cachingStrategyF'), [ - lambda: CachingStrategyJSON.instance(), - lambda: CachingStrategyPickle.instance(), - lambda: CachingStrategyPickle.instance(gzip=False) + lambda: CachingStrategy.use(JSON), + lambda: CachingStrategy.use(Pickle), + lambda: CachingStrategy.use(Pickle, gzip=False), ]) def test_cache(cachingStrategyF): cachingStrategy = cachingStrategyF() - api = Api(cachingStrategy=cachingStrategy) + api = Api() x = api.query('node/42467507') y = api.query('node/42467507') cachingStrategy.close() - api = Api(cachingStrategy=cachingStrategy) + api = Api() z = api.query('node/42467507') for a in [x, y, z]: assert a.version() >= 5 assert float(a.cacheVersion()) >= 1 assert x.cacheTimestamp() == y.cacheTimestamp() assert x.cacheTimestamp() == z.cacheTimestamp() + CachingStrategy.use(JSON)