Merge pull request #923 from openzim/upgrade_watcher

Upgrade watcher dependencies

watcher/CONTRIBUTING.md

@@ -0,0 +1,27 @@
+# Contributing
+
+In order to test this component, you need:
+- a test Zimfarm instance with a username + password
+- a test S3 bucket  (compliance must NOT be activated on this bucket)
+- credentials to access this bucket (keyId and secretAccessKey suggested below)
+- Docker
+
+Rebuild the Docker image:
+
+```
+docker build -t local-zf-watcher .
+```
+
+Export the secret `S3_URL` as environment variable. Note that the S3 URL starts with `https`.
+
+
+On Bash/Zsh shells (replace `<your_s3_host_name>`, `<your_key_id>`, `<your_secret_access_key>` and `<your_bucket>` with proper values):
+
+```
+ export S3_URL="https://<your_s3_host_name>/?keyId=<your_key_id>&secretAccessKey=<your_secret_access_key>&bucketName=<your_bucket>"
+```
+
+Run a test (here my zimfarm is running in docker on a container `backend` in network `zimfarm_default`, adapt command to your local setup):
+```
+docker run -it --rm -e ZIMFARM_API_URL=http://backend:8000/v1 -e S3_URL=$S3_URL --network zimfarm_default local-zf-watcher watcher --zimfarm-username admin --zimfarm-password admin --only tezos.stackexchange.com --runonce
+```

watcher/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-alpine
+FROM python:3.12-alpine
 
 LABEL zimfarm=true
 LABEL org.opencontainers.image.source https://github.com/openzim/zimfarm

watcher/requirements.txt

@@ -1,6 +1,6 @@
 pif==0.8.2
-requests>=2.26,<3.0
-humanfriendly>=9.2,<10.0
-PyJWT>=2.4.0,<3.0
-kiwixstorage>=0.8.2,<0.9
-xml-to-dict>=0.1.6,<0.2
+requests==2.31.0
+humanfriendly==10.0
+PyJWT==2.8.0
+kiwixstorage==0.8.3
+xml-to-dict==0.1.6

watcher/watcher.py

@@ -26,7 +26,6 @@
 import datetime
 import json
 import logging
-import multiprocessing
 import os
 import pathlib
 import re
@@ -64,21 +63,6 @@
     logging.getLogger(logger_name).setLevel(logging.WARNING)
 
 
-def is_running_inside_container():
-    """whether running inside a Docker container"""
-    fpath = pathlib.Path("/proc/self/cgroup")
-    if not fpath.exists():
-        return False
-    try:
-        with open(fpath, "r") as fh:
-            for line in fh.readlines():
-                if line.strip().rsplit(":", 1)[-1] != "/":
-                    return True
-    finally:
-        pass
-    return False
-
-
 def get_version_for(url):
     """casted datetime of the Last-Modified header for an URL"""
     with requests.head(url, allow_redirects=True) as resp:
@@ -506,7 +490,7 @@ def run(self):
 
         checked_on = datetime.datetime.now()
         self.check_and_go()
-        while self.running:
+        while self.running and not self.runonce:
             if datetime.datetime.now() > checked_on + self.duration:
                 checked_on = datetime.datetime.now()
                 self.check_and_go()
@@ -559,13 +543,10 @@ def entrypoint():
     )
     parser.add_argument(
         "--threads",
-        help="How many threads to run to parallelize download/upload? "
-        "Defaults to 1 inside Docker as we can't guess available CPUs",
+        help="How many threads to run to parallelize download/upload? Defaults to 1",
         dest="nb_threads",
         type=int,
-        default=(
-            1 if is_running_inside_container() else multiprocessing.cpu_count() - 1 or 1
-        ),
+        default=1,
     )
 
     parser.add_argument(
@@ -584,6 +565,13 @@ def entrypoint():
         "--debug", help="Enable verbose output", action="store_true", default=False
     )
 
+    parser.add_argument(
+        "--runonce",
+        help="Run only one check and stops",
+        action="store_true",
+        default=False,
+    )
+
     parser.add_argument(
         "--version",
         help="Display version and exit",