Merge branch 'feature-cache-timeout'

README.md

@@ -38,11 +38,11 @@ Main Python application to create and serve your tiles, built with Flask.
 - Set your timezone with the `TZ` environment variable to configure the scheduler, defaults to *UTC*.
 
 ### Redis JSON
-Functions as a cache and holds the scheduler data storage and history.
+Functions as a cache, holds your configurations.
 - Needs a volume at **/data** to store your configurations permanently.
 
 ## Configuration
-Create a yml config file where you have mounted your `/data/tiles.yml` folder. Take a look at the provided `tiles.example.yml` for the basic syntax. *tiles* is the top level key, list your tiles below. The main key of the tile is your slug and will become your url, so use no spaces or special characters. 
+Create a yml config file at `/data/tiles.yml`. Take a look at the provided `tiles.example.yml` for the basic syntax. *tiles* is the top level key, list your tiles below. The main key of the tile is your slug and will become your url, so use no spaces or special characters. 
 
 ### tile_name
 Give your tile a unique human readable name.
@@ -76,10 +76,18 @@ Provide your custom font by adding them to `/data/fonts`, in TTF format only and
 Defaults to `true` for all numbers. Shorten long numbers in to a more human readable string, like *14502* to *14.5K*.
 
 ### recreate: optional
-Recreate tiles periodically, provide your custom schedule as a cron tab or use `on_demand` to recreate the tile for every request. Defaults to `0 0 * * *` aka every day at midnight. Be aware of any rate limiting and API quotas you might face with a too frequent schedule. 
-Note:
-- There is automatically a random jitter for cron tab of 15 secs to avoid parallel requests for a lot of tiles.
-- There is a failsafe in place to block recreating tiles faster than every 60 seconds. 
+Set the lifetime of your tiles and define when the tile will be recreated if requested. Defaults to *1d*, e.g. recreate every day.
+
+Valid options:
+- *120*: A number indicates seconds till expire
+- *10min*: Minutes till expire
+- *2h*: Hours till expire
+- *1d*: Days till expire
+- *on_demand*: Will recreate for every request.
+
+Note: 
+- Be aware of any rate limiting and API quotas you might face with a too short expiration.
+- There is a failsafe in place to block recreating tiles faster than every 60 seconds.
 
 ## API requests
 Get values from a public API by providing the url and key_map.

tilefy/requirements.txt

@@ -3,6 +3,6 @@ beautifulsoup4==4.11.1
 flask==2.1.2
 Pillow==9.1.1
 PyYAML==6.0
-redis==4.3.3
+redis==4.3.4
 requests==2.28.0
 uwsgi==2.0.20

tilefy/src/cache.py

@@ -0,0 +1,47 @@
+"""configure scheduled jobs"""
+
+from redis.connection import ResponseError
+from src.template import create_single_tile
+from src.tilefy_redis import TilefyRedis
+
+
+class CacheManager:
+    """handle rebuild cache for tiles"""
+
+    SEC_MAP = {
+        "min": 60,
+        "h": 60 * 60,
+        "d": 60 * 60 * 24,
+    }
+
+    def __init__(self, tilename):
+        self.tilename = tilename
+        self.tile_config = self.get_tile_config()
+
+    def get_tile_config(self):
+        """get conf from redis"""
+        path = f"tiles.{self.tilename}"
+        try:
+            tile_config = TilefyRedis().get_message("config", path=path)
+        except ResponseError:
+            tile_config = False
+
+        return tile_config
+
+    def validate(self):
+        """validate cache"""
+        key = f"lock:{self.tilename}"
+        use_cached = TilefyRedis().get_message(key)
+        if use_cached:
+            print(f"{self.tilename}: use cached tile")
+            return
+
+        create_single_tile(self.tilename, self.tile_config)
+
+
+def clear_locks():
+    """clear all locks from redis"""
+    _redis = TilefyRedis()
+    all_locks = _redis.get_keys("lock")
+    for lock in all_locks:
+        _redis.del_message(lock)

tilefy/src/config_parser.py

@@ -0,0 +1,109 @@
+"""parse and load yml"""
+
+import os
+import re
+
+import yaml
+from src.cache import clear_locks
+from src.tilefy_redis import TilefyRedis
+
+
+class ConfigFile:
+    """represent tile.yml file"""
+
+    TILES_CONFIG = "/data/tiles.yml"
+    VALID_KEYS = [
+        "background_color",
+        "font_color",
+        "font",
+        "height",
+        "humanize",
+        "key_map",
+        "logos",
+        "plugin",
+        "recreate",
+        "tile_name",
+        "url",
+        "width",
+    ]
+    SEC_MAP = {
+        "min": 60,
+        "h": 60 * 60,
+        "d": 60 * 60 * 24,
+    }
+    MIN_EXPIRE = 60
+
+    def __init__(self):
+        self.exists = os.path.exists(self.TILES_CONFIG)
+        self.config_raw = False
+        self.config = False
+
+    def load_yml(self):
+        """load yml into redis"""
+        if not self.exists:
+            print("missing tiles.yml")
+            return
+
+        self.get_conf()
+        self.validate_conf()
+        self.add_expire()
+        self.save_config()
+        clear_locks()
+
+    def get_conf(self):
+        """read config file"""
+        with open(self.TILES_CONFIG, "r", encoding="utf-8") as yml_file:
+            file_content = yml_file.read()
+            self.config_raw = yaml.load(file_content, Loader=yaml.CLoader)
+
+    def validate_conf(self):
+        """check provided config file"""
+        print(f"{self.TILES_CONFIG}: validate")
+        all_tiles = self.config_raw.get("tiles")
+        if not all_tiles:
+            raise ValueError("missing tiles key")
+
+        for tile_name, tile_conf in all_tiles.items():
+            for tile_conf_key in tile_conf:
+                if tile_conf_key not in self.VALID_KEYS:
+                    message = f"{tile_name}: unexpected key {tile_conf_key}"
+                    raise ValueError(message)
+
+        self.config = self.config_raw.copy()
+
+    def add_expire(self):
+        """add expire_sec to tile_conf"""
+        all_tiles = self.config.get("tiles")
+        for tile_conf in all_tiles.values():
+            expire = self._build_expire(tile_conf)
+            tile_conf.update({"recreate_sec": expire})
+
+    def _build_expire(self, tile_config):
+        """validate config recreate return parsed secs"""
+        recreate = tile_config.get("recreate", False)
+        if not recreate:
+            return self.SEC_MAP["d"]
+
+        if isinstance(recreate, int):
+            if recreate < self.MIN_EXPIRE:
+                return self.MIN_EXPIRE
+
+            return recreate
+
+        if recreate == "on_demand":
+            return self.MIN_EXPIRE
+
+        try:
+            value, unit = re.findall(r"[a-z]+|\d+", recreate.lower())
+        except ValueError as err:
+            print(f"failed to extract value and unit of {recreate}")
+            raise err
+
+        if unit not in self.SEC_MAP:
+            raise ValueError(f"unit not in {self.SEC_MAP.keys()}")
+
+        return int(value) * self.SEC_MAP.get(unit)
+
+    def save_config(self):
+        """save config in redis"""
+        TilefyRedis().set_message("config", self.config)

tilefy/src/template.py

@@ -115,12 +115,19 @@ def create_all_tiles():
 
 def create_single_tile(tile_slug, tile_config):
     """create a single tile"""
-    key = f"lock:{tile_slug}"
-    locked = TilefyRedis().get_message(key)
-    if locked:
-        print(f"{tile_slug}: skip rebuild within 60secs")
-        return
-
     TileImage(tile_slug, tile_config).build_tile()
-    message = {"recreate": int(datetime.now().strftime("%s"))}
-    TilefyRedis().set_message(key, message, expire=60)
+
+    now = datetime.now()
+    date_format = "%Y-%m-%d %H:%M:%S"
+    expire_sec = tile_config["recreate_sec"]
+    expire_epoch = int(now.strftime("%s")) + expire_sec
+    expire_str = datetime.fromtimestamp(expire_epoch).strftime(date_format)
+
+    message = {
+        "recreated": int(now.strftime("%s")),
+        "recreated_str": now.strftime(date_format),
+        "expire": expire_epoch,
+        "expire_str": expire_str,
+    }
+
+    TilefyRedis().set_message(f"lock:{tile_slug}", message, expire=expire_sec)

tilefy/src/tilefy_redis.py

@@ -4,9 +4,6 @@
 import os
 
 import redis
-import yaml
-
-TILES_CONFIG = "/data/tiles.yml"
 
 
 class RedisBase:
@@ -41,20 +38,13 @@ def get_message(self, key, path="."):
 
         return False
 
+    def get_keys(self, key):
+        """get list of all key matches"""
+        command = f"{self.NAME_SPACE}{key}:*"
+        all_keys = self.conn.execute_command("KEYS", command)
+
+        return [i.decode().split(self.NAME_SPACE)[1] for i in all_keys]
+
     def del_message(self, key):
         """delete message from redis"""
         self.conn.execute_command("JSON.DEL", self.NAME_SPACE + key)
-
-
-def load_yml():
-    """read yml file"""
-
-    if not os.path.exists(TILES_CONFIG):
-        print("missing tiles.yml")
-        return
-
-    with open(TILES_CONFIG, "r", encoding="utf-8") as yml_file:
-        file_content = yml_file.read()
-        config_raw = yaml.load(file_content, Loader=yaml.CLoader)
-
-    TilefyRedis().set_message("config", config_raw)